From 6c2ec1c08ab424039750b446e40a94876b3978f8 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= Date: Mon, 19 Jan 2015 11:02:41 +0100 Subject: [PATCH] Documentation. --- README.markdown | 148 ++++++++++++++++++++++++++++++++++++++++++++++ manifests/accounting.pp | 2 + manifests/devel/hadoop.pp | 2 +- manifests/init.pp | 6 +- manifests/kdc.pp | 4 ++ 5 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 README.markdown diff --git a/README.markdown b/README.markdown new file mode 100644 index 0000000..642af20 --- /dev/null +++ b/README.markdown @@ -0,0 +1,148 @@ +####Table of Contents + +1. [Overview](#overview) +2. [Module Description - What the module does and why it is useful](#module-description) +3. [Setup - The basics of getting started with site\_hadoop](#setup) + * [What cesnet-hadoop module affects](#what-site_hadoop-affects) + * [Beginning with hadoop](#beginning-with-site_hadoop) +4. [Usage - Configuration options and additional functionality](#usage) +5. [Reference - An under-the-hood peek at what the module is doing and how](#reference) + * [Classes](#classes) + * [Module Parameters](#parameters) +5. [Limitations - OS compatibility, etc.](#limitations) +6. [Development - Guide for contributing to the module](#development) + + +##Overview + +Helper module for using together with CESNET Hadoop puppet modules. There are implemented decisions not meant to be directly in generic Hadoop modules: settings of Cloudera repository, installing particular version of java, desired packages, and there are custom scripts for accounting, ... + + +##Module Description + +This module performs settings and decisions not meant to be in generic Hadoop modules: +* sets Coudera repository +* installs particular version of Java +* (optionally) custom scripts for accounting +* (optionally) enable autoupdates + +Supported: + +* Debian 7/wheezy + Cloudera distribution (tested on Hadoop 2.5.0) + + +##Setup + + +###What cesnet-hadoop module affects + +* Packages: java, Kerberos, other "admin look & feel" packages (less, vim, ...), optionally cron-apt +* Files modified: + * /etc/apt/sources.list.d/cloudera.list + * /etc/apt/preferences.d/10\_cloudera.pref + * Cloudera apt gpg key + * (optionally) */etc/cron-apt/config*, */etc/cron-apt/action.d/9-upgrade*, *etc/cron.d/cron-apt* + +**Note**: Security files are NOT handled by this module. They needs to be copied to proper places for CESNET Hadoop puppet modules. + + +###Beginning with site\_hadoop + +**Example**: the basic usage, core part neccessary for cesnet-hadoop: + + class{'site_hadoop': + stage => setup, + } + +Better to set stage to 'setup', because this will set also the repository. All Hadoop puppet modules would need depend on this otherwise. + + +##Usage + +**Example 1**: enable autoupdates + + class{'site_hadoop': + email => 'valtri@civ.zcu.cz', + stage => 'setup', + } + + class{'site_hadoop::autoupdate': + time => '0 5 * * *', + } + +**Example 2**: enable Hadoop accounting: + + class { '::mysql::server': + root_password => 'strongpassword', + } + + mysql::db { 'accounting': + user => 'accounting', + password => 'accpass', + host => 'localhost', + grant => ['SELECT', 'INSERT', 'UPDATE', 'DELETE'], + sql => '/usr/local/share/hadoop/accounting.sql', + } + + class{'site_hadoop::accounting': + db_user => 'accounting', + db_password => 'accpass', + email => 'email@example.com', + accountinmg_hdfs => '0 */4 * * *', + accountinmg_quota => '0 */4 * * *', + accountinmg_jobs => '10 2 * * *', + } + + # site_hadoop::accounting provides the SQL import script + Class['site_hadoop::accounting'] -> Mysql::Db['accounting'] + # start accounting after Hadoop startup (not strictly needed) + #Class['hadoop::namenode::install'] -> Class['site_hadoop::accounting'] + + +##Reference + + +###Classes + +* devel: + * **hadoop**: Local post-installation steps for Hadoop for testing in Vagrant +* kdc: + * client + * params + * server +* **accounting**: Custom Hadoop accouting scripts +* **autoupdate**: Configure automatic updates on Debian +* cloudera +* config +* init +* install +* kdc: Experiments with KDC + + +###Module Parameters + +####`email` undef + +Email address to send errors from cron. + +####`mirror` 'cloudera' + +Cloudera mirror to use. + +Values: +* **cloudera** +* **scientific** + +Kerberos realm to use. It needs to be specified, when security is enabled. + + + +##Limitations + +Only Debian 7 fully supported. The core part will work on Fedora 21 too. + + +##Development + +* Repository: [http://scientific.zcu.cz/git/?p=meta-site\_hadoop.git;a=summary](http://scientific.zcu.cz/git/?p=meta-site_hadoop.git;a=summary) +* Email: František Dvořák <valtri@civ.zcu.cz> diff --git a/manifests/accounting.pp b/manifests/accounting.pp index 33b4307..e3e9708 100644 --- a/manifests/accounting.pp +++ b/manifests/accounting.pp @@ -1,5 +1,7 @@ # == Class site_hadoop::accounting # +# Custom Hadoop accouting scripts. +# # Requires: # * database # * hdfs user and group (=hadoop) diff --git a/manifests/devel/hadoop.pp b/manifests/devel/hadoop.pp index 56e4580..12a907d 100644 --- a/manifests/devel/hadoop.pp +++ b/manifests/devel/hadoop.pp @@ -1,5 +1,5 @@ # == Class site_hadoop::devel::hadoop -# Local post-installation steps for Hadoop. +# Local post-installation steps for Hadoop for testing in Vagrant. class site_hadoop::devel::hadoop { hadoop::kinit{'vagrant-kinit': touchfile => 'vagrant-user-created', diff --git a/manifests/init.pp b/manifests/init.pp index cae77ac..e0781b6 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -4,13 +4,11 @@ # # ##Parameters # -# ####`email` = undef -# = undef +# ####`email` undef # # Email address to send errors from cron. # -# ####`mirror` -# = 'cloudera' +# ####`mirror` 'cloudera' # # Cloudera mirror to use. # diff --git a/manifests/kdc.pp b/manifests/kdc.pp index bf43922..6896e25 100644 --- a/manifests/kdc.pp +++ b/manifests/kdc.pp @@ -1,3 +1,7 @@ +# == Class: site_hadoop::kdc +# +# Experiments with KDC. +# class site_hadoop::kdc ( $realm = $site_hadoop::kdc::params::realm, $master_password = $site_hadoop::kdc::params::master_password, -- 1.8.2.3