Added zookeeper support

README.md

@@ -1 +1,10 @@
-Various puppet scripts to setup apache tools
+# Various puppet scripts to setup apache tools
+## Setup Apache Directory (ApacheDS)
+
+    class {"plnxtools":}
+    class {"plnxtools::apacheds":}
+    plnxtools::apacheds::instance{"planix-instance-01":
+            partitionid => "planix-01",
+            partitionsuffix => "dc=uplanix,dc=org"
+    }
+

lib/facter/plnxtools_ads_instances.rb

@@ -0,0 +1,17 @@
+Facter.add('ads_instances') do
+  setcode do 
+    Dir.glob('/opt/plnxtools/ads/*.instance').map do |x|
+      content = File.read(x)
+      File.basename(x,File.extname(x))
+    end
+  end
+end
+
+Facter.add('ads_syncrepls') do
+  setcode do 
+    Dir.glob('/opt/plnxtools/ads/*.syncrepl').map do |x|
+      content = File.read(x)
+      File.basename(x,File.extname(x))
+    end
+  end
+end

manifests/apacheds.pp

@@ -13,6 +13,11 @@ inherits ::plnxtools
 {
 	$ads_home = "$opt_dir/$appname"
 
+	file {"opt_plnx_ads":
+		path => "$opt_dir/plnxtools/ads",
+		ensure => directory
+	}
+
 	package {"openldap-client":
 		ensure => installed
 	} 
@@ -35,58 +40,213 @@ inherits ::plnxtools
 	}
 }
 
+class plnxtools::apacheds::del(
+	$ensure = stopped
+)
+inherits plnxtools::apacheds
+{
+	$in = $facts['ads_instances'] 
+	$in.each |$x| {
+
+		if !defined(Plnxtools::Apacheds::Instance[$x]) {
+		        plnxtools::apacheds::instance{$x:
+		                partitionid => "$x",
+		                partitionsuffix => "dc=$x,dc=org",
+				ensure => $ensure,
+		        }
+		}
+	}
+}
 
 define plnxtools::apacheds::instance(
 	$ensure = running,
 	$ldap_port = 10389,
 	$ldaps_port = 10638,
+	$admin_password = "secret",
 	$partitionsuffix,	
 	$partitionid
 
 ) {
 	$ads_instances = $plnxtools::apacheds::ads_instances
-	$log_file = "$plnxtools::apacheds::ads_logdir/$title.log"
+	if $ensure != purged { 
+		$log_file = "$plnxtools::apacheds::ads_logdir/$title.log"
 
-	# 
-	# Create a base 64 encoded ads-contextentry 
-	# see: https://directory.apache.org/apacheds/basic-ug/1.4.3-adding-partition.html
-	#
-	$ads_contextentry0 = base64('encode',"dn: $partitionsuffix\ndc: uniadmin\nobjectclass: domain\nobjectclass: top\n\n")
-	$ads_contextentry = strip(regsubst($ads_contextentry0, '\n', "\n "))
+		# 
+		# Create a base 64 encoded ads-contextentry 
+		# see: https://directory.apache.org/apacheds/basic-ug/1.4.3-adding-partition.html
+		#
+		$ads_contextentry0 = base64('encode',"dn: $partitionsuffix\ndc: $title\nobjectclass: domain\nobjectclass: top\n\n")
+		$ads_contextentry = strip(regsubst($ads_contextentry0, '\n', "\n "))
 
 
-	file{"$ads_instances/$title":
-		ensure => directory
-	} ->
-	file{"$ads_instances/$title/conf":
-		ensure => directory
-	} ->
-	file{"$ads_instances/$title/log":
-		ensure => directory
-	} ->
-	file{"$ads_instances/$title/run":
-		ensure => directory
-	} ->
-	file{"$ads_instances/$title/conf/log4j.properties":
-		ensure => file,
-		content => template ("plnxtools/apacheds/log4j.properties.erb"),
-	}->
-	file{"/tmp/ads-$title-config.ldif":
-		ensure => file,
-		content => template ("plnxtools/apacheds/config.ldif.erb"),
-	}->
-	exec {"/bin/cp /tmp/ads-$title-config.ldif $ads_instances/$title/conf/config.ldif":
-		creates => "$ads_instances/$title/conf/config.ldif_migrated"
+		file{"$ads_instances/$title":
+			ensure => directory
+		} ->
+		file{"$ads_instances/$title/conf":
+			ensure => directory
+		} ->
+		file{"$ads_instances/$title/log":
+			ensure => directory
+		} ->
+		file{"$ads_instances/$title/run":
+			ensure => directory
+		} ->
+		file{"$ads_instances/$title/conf/log4j.properties":
+			ensure => file,
+			content => template ("plnxtools/apacheds/log4j.properties.erb"),
+		}->
+		file{"/tmp/ads-$title-config.ldif":
+			ensure => file,
+			content => template ("plnxtools/apacheds/config.ldif.erb"),
+		}->
+		exec {"/bin/cp /tmp/ads-$title-config.ldif $ads_instances/$title/conf/config.ldif":
+			creates => "$ads_instances/$title/conf/config.ldif_migrated"
+		}
+
+
+		$bin_dir = "${plnxtools::apacheds::opt_dir}/${plnxtools::apacheds::appname}/bin"
+
+		file {"/opt/plnxtools/ads/$title.instance":
+			ensure => file
+		}->
+		service {"apacheds_${title}":
+			start => "$bin_dir/apacheds.sh $title start",
+			stop => "$bin_dir/apacheds.sh $title stop",
+			status => "$bin_dir/apacheds.sh $title status | grep 'is running'",
+			ensure => $ensure,
+		}->
+		exec{"set_passwd":
+			# To sleep 20 secs is an ugly hack, better would be to wait somehow 
+			# til the port  is ready
+			command => "/bin/sleep 30 && /usr/bin/printf \
+			 'dn: uid=admin,ou=system\nchangetype: modify\nreplace: userPassword\nuserPassword: $admin_password'\
+			| ldapmodify -c -h localhost -p $ldap_port -D uid=admin,ou=system -w secret",
+			creates => "$ads_instances/$title/conf/config.ldif_migrated"
+		}
+	}
+	else{
+		$bin_dir = "${plnxtools::apacheds::opt_dir}/${plnxtools::apacheds::appname}/bin"
+		service {"apacheds_${title}":
+			start => "$bin_dir/apacheds.sh $title start",
+			stop => "$bin_dir/apacheds.sh $title stop",
+			status => "$bin_dir/apacheds.sh $title status | grep 'is running'",
+			ensure => stopped,
+		} ->
+		file {"/opt/plnxtools/ads/$title.instance":
+			ensure => absent
+		} ->
+		file{"$ads_instances/$title":
+			ensure => absent,
+			force => true
+		} 
 	}
 
-
-
-	$bin_dir = "${plnxtools::apacheds::opt_dir}/${plnxtools::apacheds::appname}/bin"
-
-	service {"apacheds_${title}":
-		start => "$bin_dir/apacheds.sh $title start",
-		stop => "$bin_dir/apacheds.sh $title stop",
-		status => "$bin_dir/apacheds.sh $title status | grep 'is running'",
-		ensure => running
-	}
+}
+
+
+
+
+define plnxtools::apacheds::syncrepl(
+	$parition,
+	$consumerid,
+	$ldap_password,
+	$ldap_port = 389,
+
+	$provider_port = 10389
+)
+{
+	$params = "\
+ads-replSearchTimeOut: 0
+ads-replSearchScope: sub
+ads-searchBaseDN: dc=example,dc=com
+ads-replProvHostName: localhost
+ads-replStrictCertValidation: FALSE
+ads-replRefreshNPersist: TRUE
+ads-replUseTls: FALSE
+ads-replUserDn: uid=admin,ou=system
+ads-replProvPort: $provider_port
+ads-replRefreshInterval: 60000
+ads-enabled: TRUE
+ads-replConsumerId: consumer1
+ads-replUserPassword: secret
+ads-replSearchSizeLimit: 0
+ads-replAttributes: *
+ads-replSearchFilter: (objectClass=*)
+ads-replAliasDerefMode: never
+objectClass: top
+objectClass: ads-base
+objectClass: ads-replConsumer
+
+"
+	$del_params = "\
+dn: ads-replConsumerId=${$title},ou=replConsumers,ads-serverId=ldapServer,ou=servers,ads-directoryServiceId=default,ou=config
+changetype: delete
+
+"
+	$add_params = "\
+dn: ads-replConsumerId=${$title},ou=replConsumers,ads-serverId=ldapServer,ou=servers,ads-directoryServiceId=default,ou=config
+changetype: add
+$params
+
+"
+
+
+
+	file {"/tmp/$title-add":
+		ensure => file,
+		content => $add_params
+	}
+	file {"/tmp/$title-del":
+		ensure => file,
+		content => $del_params
+	}
+
+	exec {"get_ldap$title":
+		command => "\
+/usr/local/bin/ldapsearch -LLL -h localhost -p 389 -D uid=admin,ou=system -w $ldap_password \
+-b ou=replConsumers,ads-serverId=ldapServer,ou=servers,ads-directoryServiceId=default,ou=config ads-replConsumerId=$title > /tmp/syncrepl-$title"
+	}
+
+$dnp="\
+dn: ads-replConsumerId=$title,ou=replConsumers,ads-serverId=ldapServer,ou=s
+ ervers,ads-directoryServiceId=default,ou=config
+"
+
+	file {"/tmp/syncrepl-$title":
+		ensure => file,
+		content => "$dnp$params",
+		require => Exec["get_ldap$title"]
+	} 
+	
+
+
+	$mps = $facts['ads_syncrepls']
+	$ldapmodify = "/usr/local/bin/ldapmodify"
+
+	if $title in $mps {
+		exec{"syscrepl_del_$title":
+			command => "$ldapmodify -c -h localhost -p $ldap_port -D uid=admin,ou=system -w $ldap_password -f /tmp/$title-del",
+			refreshonly=>true,
+			subscribe => File["/tmp/syncrepl-$title"]
+		} ->
+		exec{"syscrepl_exec_$title":
+			command => "$ldapmodify -c -h localhost -p $ldap_port -D uid=admin,ou=system -w $ldap_password -f /tmp/$title-add",
+			refreshonly=>true,
+			subscribe => File["/tmp/syncrepl-$title"]
+		} -> 
+		file {"/opt/plnxtools/ads/$title.syncrepl":
+			ensure => file
+		}
+	} else {
+
+		exec{"syscrepl_exec_$title":
+			command => "$ldapmodify -c -h localhost -p $ldap_port -D uid=admin,ou=system -w $ldap_password -f /tmp/$title-add",
+			refreshonly=>true,
+			subscribe => File["/tmp/syncrepl-$title"]
+		} -> 
+		file {"/opt/plnxtools/ads/$title.syncrepl":
+			ensure => file
+		}
+		
+	}		
 }

manifests/init.pp

@@ -17,6 +17,10 @@ inherits plnxtools::params{
 	file {"opt_dir":
 		path => $opt_dir,
 		ensure => directory
+	}->
+	file {"opt_dir_plnx":
+		path => "$opt_dir/plnxtools",
+		ensure => directory
 	}
 
 	# 

manifests/nutch/params.pp

@@ -2,7 +2,7 @@
 # params.pp
 #
 
-class plnxtools::nutch::params{
+class plnxtools::zookeeper::params{
 	$distfile = "apache-nutch-1.17-bin.tar.gz"
 	$srcdir = "https://downloads.apache.org/nutch/1.17/"
 	$checksum = "ab36cd1c158c9e0807313a103189588ccd9a0cd0fa990a0f8bb482dd078f6d701cc5369c3ce0fcac021ed2bc16614572b15b12b5d6929a9085cd43bd8e549fe9"

manifests/params.pp

@@ -12,6 +12,16 @@ class plnxtools::params{
 	}
 	default: {
 
-	}} 
+	}}
+	$logdir = "/var/log"
+
+
+  $zookeeper_version = "3.6.3"
+  $zookeeper_sourcesite = "https://downloads.apache.org"
+  $zookeeper_distfile = "apache-zookeeper-$zookeeper_version-bin.tar.gz"
+  $zookeeper_dirname = "apache-zookeeper-$zookeeper_version-bin"
+	$zookeeper_sourcedir = "$zookeeper_sourcesite/zookeeper/zookeeper-$zookeeper_version"
+
+	$zookeeper_instances = "/var/zookeeper"
 
 }

manifests/zookeeper.pp

@@ -4,13 +4,97 @@
 
 
 class plnxtools::zookeeper
-() inherits plnxtools::nutch::params
+(
+	$appname = "zookeeper",
+  $zookeeper_logdir = "$::plnxtools::params::logdir/$appname"
+
+) inherits plnxtools::params
 {
-	class {"plnxtools": } 
-	->
 	plnxtools::install{"zookeeper":
-		sourcedir => "https://downloads.apache.org/zookeeper/zookeeper-3.6.1",
-		distfile => "apache-zookeeper-3.6.1-bin.tar.gz",
-		dirname => "apache-zookeeper-3.6.1-bin",
+		sourcedir => $zookeeper_sourcedir,
+		distfile => $zookeeper_distfile,
+		dirname => $zookeeper_dirname,
+	} ->
+	file {"$zookeeper_instances":
+		ensure => directory
+	}
+	file {"$zookeeper_logdir":
+		ensure => directory
 	}
 }
+
+
+define plnxtools::zookeeper::instancedir
+( $dirname,
+  $log_file)
+{
+		file{"$dirname/$title":
+			ensure => directory
+		} ->
+		file{"$dirname/$title/conf":
+			ensure => directory
+		} ->
+		file{"$dirname/$title/log":
+			ensure => directory
+		} ->
+		file{"$dirname/$title/run":
+			ensure => directory
+		} ->
+		file{"$dirname/$title/data":
+			ensure => directory
+		} 
+		file{"$dirname/$title/conf/log4j.properties":
+			ensure => file,
+			content => template ("plnxtools/log4j.properties.erb"),
+		}
+}
+
+define plnxtools::zookeeper::instance(
+	$ensure = running,
+
+  $servers = {
+    '1' => "localhost:2888:3888"
+  },
+
+  $myid = '1',
+  $client_port = '2181'
+
+
+) {
+  $zookeeper_instances = $plnxtools::zookeeper::zookeeper_instances
+	if $ensure != purged { 
+		$log_file = "$plnxtools::zookeeper::zookeeper_logdir/$title.log"
+    $conf_dir = "$zookeeper_instances/$title/conf"
+    $conf_file = "$conf_dir/zoo.cfg"
+    $data_dir = "$zookeeper_instances/$title/data"
+    $myid_file = "$data_dir/myid"
+		$bin_dir = "${plnxtools::zookeeper::opt_dir}/${plnxtools::zookeeper::appname}/bin"
+
+    plnxtools::zookeeper::instancedir {"$title":
+      dirname => $zookeeper_instances,
+      log_file => $log_file,
+
+    } ->
+    file {$conf_file:
+      ensure =>  file,
+      content => template("plnxtools/zookeeper/zoo.cfg.erb"),
+    }->
+    file {$myid_file:
+      ensure   => file,
+      content => $myid
+    }
+    service {"zookeeper_${title}":
+      start     => "$bin_dir/zkServer.sh --config $conf_dir start",
+      stop      => "$bin_dir/zkServer.sh --config $conf_dir stop",
+      status    => "/bin/kill -0 `cat $data_dir/zookeeper_server.pid`",
+      ensure    => $ensure,
+      subscribe =>  File[$conf_file],
+    }
+
+  }
+
+}
+
+
+
+

templates/apacheds/config.ldif.erb

@@ -705,7 +705,7 @@ objectclass: organizationalUnit
 objectclass: top
 
 dn: ads-transportid=ldap,ou=transports,ads-serverId=ldapServer,ou=servers,ads-directoryServiceId=default,ou=config
-ads-systemport: 10389
+ads-systemport: <%= @ldap_port %>
 ads-transportnbthreads: 8
 ads-transportaddress: 0.0.0.0
 ads-transportid: ldap

templates/log4j.properties.erb

@@ -0,0 +1,17 @@
+# 
+# managed by puppet
+#
+
+log4j.rootCategory=WARN, F, stdout
+
+log4j.appender.F=org.apache.log4j.RollingFileAppender
+log4j.appender.F.File=<%= @log_file %>
+log4j.appender.F.MaxFileSize=100MB
+log4j.appender.F.MaxBackupIndex=3
+log4j.appender.F.layout=org.apache.log4j.PatternLayout
+log4j.appender.F.layout.ConversionPattern=[%d{HH:mm:ss}] %p [%c] - %m%n
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d{HH:mm:ss}] %p [%c] - %m%n
+

templates/zookeeper/configuration.xsl

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:template match="configuration">
+<html>
+<body>
+<table border="1">
+<tr>
+ <td>name</td>
+ <td>value</td>
+ <td>description</td>
+</tr>
+<xsl:for-each select="property">
+<tr>
+  <td><a name="{name}"><xsl:value-of select="name"/></a></td>
+  <td><xsl:value-of select="value"/></td>
+  <td><xsl:value-of select="description"/></td>
+</tr>
+</xsl:for-each>
+</table>
+</body>
+</html>
+</xsl:template>
+</xsl:stylesheet>

templates/zookeeper/log4j.properties

@@ -0,0 +1,82 @@
+# Copyright 2012 The Apache Software Foundation
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+zookeeper.root.logger=INFO, CONSOLE
+
+zookeeper.console.threshold=INFO
+
+zookeeper.log.dir=.
+zookeeper.log.file=zookeeper.log
+zookeeper.log.threshold=INFO
+zookeeper.log.maxfilesize=256MB
+zookeeper.log.maxbackupindex=20
+
+zookeeper.tracelog.dir=${zookeeper.log.dir}
+zookeeper.tracelog.file=zookeeper_trace.log
+
+log4j.rootLogger=${zookeeper.root.logger}
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this 
+#
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold}
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
+
+#
+# Add ROLLINGFILE to rootLogger to get log file output
+#
+log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
+log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold}
+log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file}
+log4j.appender.ROLLINGFILE.MaxFileSize=${zookeeper.log.maxfilesize}
+log4j.appender.ROLLINGFILE.MaxBackupIndex=${zookeeper.log.maxbackupindex}
+log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
+
+#
+# Add TRACEFILE to rootLogger to get log file output
+#    Log TRACE level and above messages to a log file
+#
+log4j.appender.TRACEFILE=org.apache.log4j.FileAppender
+log4j.appender.TRACEFILE.Threshold=TRACE
+log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file}
+
+log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout
+### Notice we are including log4j's NDC here (%x)
+log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n
+#
+# zk audit logging
+#
+zookeeper.auditlog.file=zookeeper_audit.log
+zookeeper.auditlog.threshold=INFO
+audit.logger=INFO, RFAAUDIT
+log4j.logger.org.apache.zookeeper.audit.Log4jAuditLogger=${audit.logger}
+log4j.additivity.org.apache.zookeeper.audit.Log4jAuditLogger=false
+log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAAUDIT.File=${zookeeper.log.dir}/${zookeeper.auditlog.file}
+log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.RFAAUDIT.Threshold=${zookeeper.auditlog.threshold}
+
+# Max log file size of 10MB
+log4j.appender.RFAAUDIT.MaxFileSize=10MB
+log4j.appender.RFAAUDIT.MaxBackupIndex=10

templates/zookeeper/zoo.cfg.erb

@@ -0,0 +1,40 @@
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial 
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between 
+# sending a request and getting an acknowledgement
+syncLimit=5
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just 
+# example sakes.
+dataDir=<%=@data_dir%>
+# the port at which the clients will connect
+clientPort=<%=@client_port%>
+# the maximum number of client connections.
+# increase this if you need to handle more clients
+#maxClientCnxns=60
+#
+# Be sure to read the maintenance section of the 
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+#autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to "0" to disable auto purge feature
+#autopurge.purgeInterval=1
+
+## Metrics Providers
+#
+# https://prometheus.io Metrics Exporter
+#metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
+#metricsProvider.httpPort=7000
+#metricsProvider.exportJvmInfo=true
+
+
+<% @servers.each do |id,val| -%>
+server.<%= id %>=<%= val %>
+<% end -%>

templates/zookeeper/zoo_sample.cfg

@@ -0,0 +1,36 @@
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial 
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between 
+# sending a request and getting an acknowledgement
+syncLimit=5
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just 
+# example sakes.
+dataDir=/tmp/zookeeper
+# the port at which the clients will connect
+clientPort=2181
+# the maximum number of client connections.
+# increase this if you need to handle more clients
+#maxClientCnxns=60
+#
+# Be sure to read the maintenance section of the 
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+#autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to "0" to disable auto purge feature
+#autopurge.purgeInterval=1
+
+## Metrics Providers
+#
+# https://prometheus.io Metrics Exporter
+#metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
+#metricsProvider.httpPort=7000
+#metricsProvider.exportJvmInfo=true
+