Merge branch 'master' into dist/eole/2.6.2/master

dicos/90_redis.xml

@@ -3,11 +3,13 @@
     <files>
         <!-- System configuration -->
         <file filelist='redis' name='/etc/redis/redis.conf' mkdir='True' rm='True'/>
+        <file filelist='redisCl' name='/etc/redis/cluster.conf' source='redis-cluster.conf' mkdir='True' rm='True'/>
         <service servicelist="svredis">redis-server</service>
         <service_access service='redis-server'>
             <port service_accesslist='saRedis' protocol='tcp' port_type='SymLinkOption'>redisPort</port>
+			<tcpwrapper>redis-server</tcpwrapper>
         </service_access>
-        <service_restriction service='ead_web'>
+        <service_restriction service='redis-server'>
                <ip interface='auto' netmask='rdClmask' netmask_type='SymLinkOption' ip_type='SymLinkOption'>rdClIP</ip>
         </service_restriction>
     </files>
@@ -21,9 +23,9 @@
             <variable name='redisMode' type='string' description="Mode d'utilisation de Redis">
                 <value>Local</value>
             </variable>
-			<variable name='redisPort' type='string' description="Port d'écoute du service Redis">
-				<value>6379</value>
-			</variable>
+            <variable name='redisPort' type='string' description="Port d'écoute du service Redis">
+                <value>6379</value>
+            </variable>
             <variable name='redisMaxMemory' type='number' description="Quantité de mémoire utilisable par Redis en Mo">
                 <value>512</value>
             </variable>
@@ -36,18 +38,20 @@
             <variable name='rdClIP' type='ip' description="Adresse IP réseau autorisée a utiliser le service Redis" multi='True'/>
             <variable name='rdClmask' type='ip' description="Masque de sous réseau"/>
 
-
             <!-- Cluster configration Part -->
             <variable name='redisRole' type='string' description="Role du serveur Redis">
                 <value>Leader</value>
             </variable>
-            <variable name='redisMasterIP' type='ip' description="Adresse IP du Leader Redis"/>
-            <variable name='redisMasterPort' type='number' description="Port d'écoute du Leader Redis"/>
-            <variable name='redisMasterPassword' type='string' description="Passphrase d'accès à la grappe Redis"/>
+            <variable name='rdClMember' type='string' description="Nom du noeud" multi='True'/>
+            <variable name='rdClMemberIP' type='ip' description="Adresse IP du noeud" multi='True'/>
+            <variable name='rdClMemberPort' type='number' description="Port d'écoute du noeud" multi='True'/>
+            <variable name='rdClMemberRole' type='ip' description="Rôle du membre"/>
+            <variable name='rdClMemberMaster' type='string' description="Noeud source des données pour ce membre"/>
         </family>
         <separators>
             <separator name='rdClIP'>Autorisation d'accès au service Redis</separator>
             <separator name='redisRole'>Grappe Redis</separator>
+            <separator name='rdClMemberIP'>Serveurs membres de la grappe Redis</separator>
         </separators>
     </variables>
     <constraints>
@@ -62,26 +66,43 @@
         <condition name='disabled_if_in' source='redisMode'>
             <param>Local</param>
             <target type='variable'>redisRole</target>
-            <target type='variable'>redisMasterIP</target>
-            <target type='variable'>redisMasterPort</target>
-            <target type='variable'>redisMasterPassword</target>
+            <target type='filelist'>redisCl</target>
         </condition>
 
         <condition name='disabled_if_in' source='redisRole'>
-            <param>Leader</param>
-            <target type='variable'>redisMasterIP</target>
-            <target type='variable'>redisMasterPort</target>
+            <param>Node</param>
+            <target type='filelist'>redisCl</target>
+			<target type='variable'>rdClMember</target>
+			<target type='variable'>rdClMemberIP</target>
+			<target type='variable'>rdClMemberPort</target>
+			<target type='variable'>rdClMemberRole</target>
+			<target type='variable'>rdClMemberMaster</target>
+        </condition>
+
+        <condition name='disabled_if_in' source='rdClMemberRole'>
+            <param>master</param>
+            <target type='variable'>rdClMemberMaster</target>
         </condition>
 
         <group master='rdClIP'>
             <slave>rdClmask</slave>
         </group>
+
+        <group master='rdClMember'>
+            <slave>rdClMemberIP</slave>
+            <slave>rdClMemberPort</slave>
+            <slave>rdClMemberRole</slave>
+            <slave>rdClMemberMaster</slave>
+        </group>
         <check name="valid_enum" target="redisRole">
             <param>['Leader','Node']</param>
         </check>
         <check name="valid_enum" target="redisMode">
             <param>['Local','Cluster']</param>
         </check>
+        <check name="valid_enum" target="rdClMemberRole">
+            <param>['master','slave']</param>
+        </check>
         <check name="valid_enum" target="redisMemoryPolicy">
             <param>['noeviction', 'allkeys-lru','volatile-lru','volatile-random', 'allkeys-random','volatile-ttl']</param>
         </check>

postservice/90-redis-init

@@ -0,0 +1,193 @@
+#!/bin/bash
+
+#
+# Cluster init !
+#
+
+#function ProgressBar {
+#    # Process data
+#    let _progress=(${1}*100/${2}*100)/100
+#    let _done=(${_progress}*4)/10
+#    let _left=40-$_done
+#    # Build progressbar string lengths
+#    _fill=$(printf "%${_done}s")
+#    _empty=$(printf "%${_left}s")
+#
+#    # 1.2 Build progressbar strings and print the ProgressBar line
+#    # 1.2.1 Output example:
+#    # 1.2.1.1 Progress : [########################################] 100%
+#    printf "\rProgress : [${_fill// /#}${_empty// /-}] ${_progress}%%"
+#}
+
+function redisRun()
+{
+	cmd="redis-cli"
+    host=$1
+    shift
+    port=$1
+    shift
+    #pass=$1
+    #shift
+
+	#opt="-h ${host} -p ${port} -a ${pass}"
+	opt="-h ${host} -p ${port}"
+	act=${@}
+	${cmd} ${opt} ${act}
+	return ${?}
+}
+#
+# Get Node ID with IP Adress and Port Number
+# Params :
+#  - $1 => Leader IP
+#  - $2 => Leader Port
+#  - $3 => Cluster Password
+#  - $4 => Node IP Adress
+#  - $5 => Node Port
+#
+function getNodeID()
+{
+    if [[ ${1} == ${3} ]]
+    then
+        searchCmd="awk '/myself,master/ { print \$1  }'"
+    else
+        searchCmd="awk '/${3}:${4}@/ { print \$1  }'"
+    fi
+    redisRun $1 $2 cluster nodes | eval ${searchCmd}
+    return ${?}
+}
+
+function redisClusterForgetAll()
+{
+    local res=0
+    for node in $(redisRun ${1} ${2} ${3} cluster nodes | awk '!/myself,master/ {print $1}')
+    do
+        redisRun ${1} ${2} ${3} "cluster forget ${node}"
+        res=$((res+${?}))
+    done
+    return ${res}
+}
+
+#
+# Assing redis slots to master Node
+# Params :
+#  - $1 => Node IP
+#  - $2 => Node Port
+#  - $3 => Cluster Password
+#  - $4 => First slot
+#  - $5 => Last slot
+#
+function redisSlotAssign()
+{
+    res=0
+    for slot in $(seq ${3} ${4})
+    do
+        OUT=$(redisRun ${1} ${2} "cluster ADDSLOTS $slot")
+        res=$((res+${?}))
+        if [[ ${OUT} =~ ERR ]]
+        then
+            OUT=$(redisRun ${1} ${2} "cluster ADDSLOTS $slot")
+        fi
+    done
+    return ${res}
+}
+
+function redisAddReplica()
+{
+    redisRun ${1} ${2} "cluster replicate ${3}"
+    return ${?}
+}
+
+CONF="/etc/redis/cluster.conf"
+#CONF=/tmp/cluster.conf
+
+[[ ! -e ${CONF} ]] && exit 0
+
+declare -A IPADDRS
+declare -A PORTS
+declare -A ROLES
+declare -A MASTERS
+declare -A NIDS
+index=0
+
+while read line
+do
+	line=${line//::/:none:}
+	li=(${line//:/ })
+
+    name=${li[0]}
+    ip=${li[1]}
+    port=${li[2]}
+    role=${li[3]}
+    master=${li[4]}
+
+    NAMES+=(${name})
+    IPADDRS+=([${name}]=${ip})
+	PORTS+=([${name}]=${port})
+	ROLES+=([${name}]=${role})
+	MASTERS+=([${name}]=${master})
+done < ${CONF}
+
+if [[ ${1} == "forget" ]]
+then
+    redisClusterForgetAll ${IPADDRS["Leader"]} ${PORTS["Leader"]}
+    exit ${?}
+fi
+
+echo " * Organise Cluster Meeting."
+for node in ${NAMES[@]}
+do
+	act="cluster meet ${IPADDRS[${node}]} ${PORTS[${node}]}"
+    echo "   - ${node} - ${IPADDRS[${node}]}:${PORTS[${node}]} meeting with :"
+    for oth in ${NAMES[@]}
+    do
+        [[ ${oth} == ${node} ]] && continue
+        echo -en "      -> ${oth} - ${IPADDRS[${oth}]} ${PORTS[${oth}]} "
+        redisRun ${IPADDRS[${oth}]} ${PORTS[${oth}]} "${act}"
+    done
+done
+
+nbMaster=$(grep -c ":master" ${CONF})
+totalslots=16384
+perNodeSlots=$((totalslots/nbMaster))
+startSlot=0
+
+echo " * Assign Slots to the nodes."
+for node in ${NAMES[@]}
+do
+    if [[ ${ROLES[$node]} == "master" ]]
+    then
+        if [[ ${startSlot} -eq 0 ]]
+        then
+            lastSlot=$(((perNodeSlots+startSlot)-1))
+        else
+            lastSlot=$((perNodeSlots+startSlot))
+        fi
+        [[ ${lastSlot} -ge ${totalslots} ]] && lastSlot=$((totalslots-1))
+        echo "    - Slots ${startSlot} to ${lastSlot} => ${node} - ${IPADDRS[${node}]} ${PORTS[${node}]}"
+        redisSlotAssign ${IPADDRS[${node}]} ${PORTS[${node}]}  ${startSlot} ${lastSlot}
+        startSlot=$((lastSlot+1))
+    fi
+done
+
+echo " * Configuring replication."
+NIDS+=()
+for node in ${NAMES[@]}
+do
+   NIDS+=([${node}]=$(getNodeID ${IPADDRS["Leader"]} ${PORTS["Leader"]} ${IPADDRS[${node}]} ${PORTS[${node}]}))
+done
+
+for node in ${NAMES[@]}
+do
+    [[ ${MASTERS[$node]} == "" ]] && continue
+    echo -ne "   - Replicate ${MASTERS[${node}]} to ${node}"
+    redisAddReplica ${IPADDRS[${node}]} ${PORTS[${node}]} ${NIDS[${MASTERS[${node}]}]}
+done
+
+echo
+echo "Cluster status :"
+echo
+redisRun ${IPADDRS["Leader"]} ${PORTS["Leader"]} cluster info
+echo
+
+
+exit 0

tmpl/redis-cluster.conf

@@ -0,0 +1,10 @@
+%if %%getVar('activer_redis','non') == 'oui' and %%getVar('redisRole','Node') == 'Leader'
+Leader:%%adresse_ip_eth0:%%{redisPort}:master::
+   %for %%node in %%getVar('rdClMember',[])
+      %if %%node.rdClMemberRole == 'master'
+%%{node}:%%{node.rdClMemberIP}:%%{node.rdClMemberPort}:%%{node.rdClMemberRole}::
+      %else
+%%{node}:%%{node.rdClMemberIP}:%%{node.rdClMemberPort}:%%{node.rdClMemberRole}:%%{node.rdClMemberMaster}:
+      %end if
+   %end for
+%end if

tmpl/redis.conf

@@ -77,7 +77,7 @@ bind 0.0.0.0
 # you are sure you want clients from other hosts to connect to Redis
 # even if no authentication is configured, nor a specific set of interfaces
 # are explicitly listed using the "bind" directive.
-protected-mode yes
+protected-mode no
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
@@ -246,225 +246,6 @@ dbfilename dump.rdb
 # Note that you must specify a directory here, not a file name.
 dir /var/lib/redis
 
-%if %%getVar('redisRole','Leader') == "Node"
-################################# REPLICATION #################################
-
-# Master-Slave replication. Use slaveof to make a Redis instance a copy of
-# another Redis server. A few things to understand ASAP about Redis replication.
-#
-# 1) Redis replication is asynchronous, but you can configure a master to
-#    stop accepting writes if it appears to be not connected with at least
-#    a given number of slaves.
-# 2) Redis slaves are able to perform a partial resynchronization with the
-#    master if the replication link is lost for a relatively small amount of
-#    time. You may want to configure the replication backlog size (see the next
-#    sections of this file) with a sensible value depending on your needs.
-# 3) Replication is automatic and does not need user intervention. After a
-#    network partition slaves automatically try to reconnect to masters
-#    and resynchronize with them.
-#
-slaveof %%redisMasterIP %%redisMasterPort
-
-# If the master is password protected (using the "requirepass" configuration
-# directive below) it is possible to tell the slave to authenticate before
-# starting the replication synchronization process, otherwise the master will
-# refuse the slave request.
-#
-masterauth %%redisMasterPassword
-
-# When a slave loses its connection with the master, or when the replication
-# is still in progress, the slave can act in two different ways:
-#
-# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
-#    still reply to client requests, possibly with out of date data, or the
-#    data set may just be empty if this is the first synchronization.
-#
-# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
-#    an error "SYNC with master in progress" to all the kind of commands
-#    but to INFO and SLAVEOF.
-#
-slave-serve-stale-data yes
-
-# You can configure a slave instance to accept writes or not. Writing against
-# a slave instance may be useful to store some ephemeral data (because data
-# written on a slave will be easily deleted after resync with the master) but
-# may also cause problems if clients are writing to it because of a
-# misconfiguration.
-#
-# Since Redis 2.6 by default slaves are read-only.
-#
-# Note: read only slaves are not designed to be exposed to untrusted clients
-# on the internet. It's just a protection layer against misuse of the instance.
-# Still a read only slave exports by default all the administrative commands
-# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve
-# security of read only slaves using 'rename-command' to shadow all the
-# administrative / dangerous commands.
-slave-read-only yes
-
-# Replication SYNC strategy: disk or socket.
-#
-# -------------------------------------------------------
-# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY
-# -------------------------------------------------------
-#
-# New slaves and reconnecting slaves that are not able to continue the replication
-# process just receiving differences, need to do what is called a "full
-# synchronization". An RDB file is transmitted from the master to the slaves.
-# The transmission can happen in two different ways:
-#
-# 1) Disk-backed: The Redis master creates a new process that writes the RDB
-#                 file on disk. Later the file is transferred by the parent
-#                 process to the slaves incrementally.
-# 2) Diskless: The Redis master creates a new process that directly writes the
-#              RDB file to slave sockets, without touching the disk at all.
-#
-# With disk-backed replication, while the RDB file is generated, more slaves
-# can be queued and served with the RDB file as soon as the current child producing
-# the RDB file finishes its work. With diskless replication instead once
-# the transfer starts, new slaves arriving will be queued and a new transfer
-# will start when the current one terminates.
-#
-# When diskless replication is used, the master waits a configurable amount of
-# time (in seconds) before starting the transfer in the hope that multiple slaves
-# will arrive and the transfer can be parallelized.
-#
-# With slow disks and fast (large bandwidth) networks, diskless replication
-# works better.
-repl-diskless-sync no
-
-# When diskless replication is enabled, it is possible to configure the delay
-# the server waits in order to spawn the child that transfers the RDB via socket
-# to the slaves.
-#
-# This is important since once the transfer starts, it is not possible to serve
-# new slaves arriving, that will be queued for the next RDB transfer, so the server
-# waits a delay in order to let more slaves arrive.
-#
-# The delay is specified in seconds, and by default is 5 seconds. To disable
-# it entirely just set it to 0 seconds and the transfer will start ASAP.
-repl-diskless-sync-delay 5
-
-# Slaves send PINGs to server in a predefined interval. It's possible to change
-# this interval with the repl_ping_slave_period option. The default value is 10
-# seconds.
-#
-# repl-ping-slave-period 10
-
-# The following option sets the replication timeout for:
-#
-# 1) Bulk transfer I/O during SYNC, from the point of view of slave.
-# 2) Master timeout from the point of view of slaves (data, pings).
-# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings).
-#
-# It is important to make sure that this value is greater than the value
-# specified for repl-ping-slave-period otherwise a timeout will be detected
-# every time there is low traffic between the master and the slave.
-#
-# repl-timeout 60
-
-# Disable TCP_NODELAY on the slave socket after SYNC?
-#
-# If you select "yes" Redis will use a smaller number of TCP packets and
-# less bandwidth to send data to slaves. But this can add a delay for
-# the data to appear on the slave side, up to 40 milliseconds with
-# Linux kernels using a default configuration.
-#
-# If you select "no" the delay for data to appear on the slave side will
-# be reduced but more bandwidth will be used for replication.
-#
-# By default we optimize for low latency, but in very high traffic conditions
-# or when the master and slaves are many hops away, turning this to "yes" may
-# be a good idea.
-repl-disable-tcp-nodelay no
-
-# Set the replication backlog size. The backlog is a buffer that accumulates
-# slave data when slaves are disconnected for some time, so that when a slave
-# wants to reconnect again, often a full resync is not needed, but a partial
-# resync is enough, just passing the portion of data the slave missed while
-# disconnected.
-#
-# The bigger the replication backlog, the longer the time the slave can be
-# disconnected and later be able to perform a partial resynchronization.
-#
-# The backlog is only allocated once there is at least a slave connected.
-#
-# repl-backlog-size 1mb
-
-# After a master has no longer connected slaves for some time, the backlog
-# will be freed. The following option configures the amount of seconds that
-# need to elapse, starting from the time the last slave disconnected, for
-# the backlog buffer to be freed.
-#
-# A value of 0 means to never release the backlog.
-#
-# repl-backlog-ttl 3600
-
-# The slave priority is an integer number published by Redis in the INFO output.
-# It is used by Redis Sentinel in order to select a slave to promote into a
-# master if the master is no longer working correctly.
-#
-# A slave with a low priority number is considered better for promotion, so
-# for instance if there are three slaves with priority 10, 100, 25 Sentinel will
-# pick the one with priority 10, that is the lowest.
-#
-# However a special priority of 0 marks the slave as not able to perform the
-# role of master, so a slave with priority of 0 will never be selected by
-# Redis Sentinel for promotion.
-#
-# By default the priority is 100.
-slave-priority 100
-
-# It is possible for a master to stop accepting writes if there are less than
-# N slaves connected, having a lag less or equal than M seconds.
-#
-# The N slaves need to be in "online" state.
-#
-# The lag in seconds, that must be <= the specified value, is calculated from
-# the last ping received from the slave, that is usually sent every second.
-#
-# This option does not GUARANTEE that N replicas will accept the write, but
-# will limit the window of exposure for lost writes in case not enough slaves
-# are available, to the specified number of seconds.
-#
-# For example to require at least 3 slaves with a lag <= 10 seconds use:
-#
-# min-slaves-to-write 3
-# min-slaves-max-lag 10
-#
-# Setting one or the other to 0 disables the feature.
-#
-# By default min-slaves-to-write is set to 0 (feature disabled) and
-# min-slaves-max-lag is set to 10.
-
-# A Redis master is able to list the address and port of the attached
-# slaves in different ways. For example the "INFO replication" section
-# offers this information, which is used, among other tools, by
-# Redis Sentinel in order to discover slave instances.
-# Another place where this info is available is in the output of the
-# "ROLE" command of a masteer.
-#
-# The listed IP and address normally reported by a slave is obtained
-# in the following way:
-#
-#   IP: The address is auto detected by checking the peer address
-#   of the socket used by the slave to connect with the master.
-#
-#   Port: The port is communicated by the slave during the replication
-#   handshake, and is normally the port that the slave is using to
-#   list for connections.
-#
-# However when port forwarding or Network Address Translation (NAT) is
-# used, the slave may be actually reachable via different IP and port
-# pairs. The following two options can be used by a slave in order to
-# report to its master a specific set of IP and port, so that both INFO
-# and ROLE will report those values.
-#
-# There is no need to use both the options if you need to override just
-# the port or the IP address.
-#
-# slave-announce-ip 5.5.5.5
-# slave-announce-port 1234
-%end if
 ################################## SECURITY ###################################
 
 # Require clients to issue AUTH <PASSWORD> before processing any other
@@ -478,7 +259,7 @@ slave-priority 100
 # 150k passwords per second against a good box. This means that you should
 # use a very strong password otherwise it will be very easy to break.
 #
-requirepass %%redisMasterPassword
+#requirepass redisMasterPassword
 
 # Command renaming.
 #