CI: Initial commit for moving CI into source repo

Create tests/CI directory and add files used by Jenkins to
build and test Asterisk.

With this commit, Jenkins will run the Asterisk Unit Tests using
the Jenkinsfile at tests/CI/unittests.jenkinsfile.  Bash scripts
to do the actual building and testing are also in the same directory.
Output is placed in tests/CI/output so that directory has been
added to .gitignore.

Change-Id: I9448065465e6de2b878634510ace8fd1ef378608

.gitignore

@@ -36,3 +36,5 @@ latex
 doxygen.log
 out/
 *.orig
+tests/CI/output
+

tests/CI/buildAsterisk.sh

+#!/bin/bash
+
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+
+gen_cats() {
+	set +x
+	action=$1
+	shift
+	cats=$@
+
+	for x in $cats ; do
+		echo " --${action}-category ${x}"
+	done
+}
+
+gen_mods() {
+	set +x
+	action=$1
+	shift
+	mods=$@
+
+	for x in $mods ; do
+		echo " --${action} ${x}"
+	done
+}
+
+sudo mkdir -p /srv/cache/externals /srv/cache/sounds || :
+sudo chown -R jenkins:users /srv/cache
+[ ! -d tests/CI/output ] && mkdir tests/CI/output
+sudo chown -R jenkins:users tests/CI/output
+
+MAKE=`which make`
+printenv | sort
+
+common_config_args="--sysconfdir=/etc --with-pjproject-bundled"
+common_config_args+=" --with-sounds-cache=/srv/cache/sounds --with-externals-cache=/srv/cache/externals"
+common_config_args+=" --enable-dev-mode"
+export WGET_EXTRA_ARGS="--quiet"
+
+runner ./configure ${common_config_args} CCACHE_DISABLE=1 >tests/CI/output/configure.txt
+
+runner ${MAKE} menuselect.makeopts
+
+runner menuselect/menuselect `gen_mods enable DONT_OPTIMIZE BETTER_BACKTRACES MALLOC_DEBUG DO_CRASH TEST_FRAMEWORK` menuselect.makeopts
+runner menuselect/menuselect `gen_mods disable COMPILE_DOUBLE BUILD_NATIVE` menuselect.makeopts
+
+cat_enables="MENUSELECT_BRIDGES MENUSELECT_CEL MENUSELECT_CDR"
+cat_enables+=" MENUSELECT_CHANNELS MENUSELECT_CODECS MENUSELECT_FORMATS MENUSELECT_FUNCS"
+cat_enables+=" MENUSELECT_PBX MENUSELECT_RES MENUSELECT_UTILS MENUSELECT_TESTS"
+runner menuselect/menuselect `gen_cats enable $cat_enables` menuselect.makeopts
+
+mod_disables="res_digium_phone chan_vpb"
+[ "$BRANCH_NAME" == "master" ] && mod_disables+=" codec_opus codec_silk codec_g729a codec_siren7 codec_siren14"
+runner menuselect/menuselect `gen_mods disable $mod_disables` menuselect.makeopts
+
+mod_enables="app_voicemail app_directory FILE_STORAGE"
+mod_enables+=" res_mwi_external res_ari_mailboxes res_mwi_external_ami res_stasis_mailbox"
+mod_enables+=" CORE-SOUNDS-EN-GSM MOH-OPSOUND-GSM EXTRA-SOUNDS-EN-GSM"
+runner menuselect/menuselect `gen_mods enable $mod_enables` menuselect.makeopts
+
+runner ${MAKE} -j8 || runner ${MAKE} -j1 NOISY_BUILD=yes
+
+ALEMBIC=$(which alembic 2>/dev/null || : )
+if [ x"$ALEMBIC" = x ] ; then
+	echo "Alembic not installed"
+	exit 1
+fi
+
+cd contrib/ast-db-manage
+find -name *.pyc -delete
+out=$(alembic -c config.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+	>&2 echo "Alembic branches were found for config"
+	>&2 echo $out
+	exit 1
+else
+	>&2 echo "Alembic for 'config' OK"
+fi
+
+out=$(alembic -c cdr.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+	>&2 echo "Alembic branches were found for cdr"
+	>&2 echo $out
+	exit 1
+else
+	>&2 echo "Alembic for 'cdr' OK"
+fi
+
+out=$(alembic -c voicemail.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+	>&2 echo "Alembic branches were found for voicemail"
+	>&2 echo $out
+	exit 1
+else
+	>&2 echo "Alembic for 'voicemail' OK"
+fi
+
+if [ -f "doc/core-en_US.xml" ] ; then
+	${MAKE} validate-docs || ${MAKE} NOISY_BUILD=yes validate-docs
+fi
+
+

tests/CI/ci.functions

+#
+# This file contains useful Bash functions
+# and can be "source"d from the scripts.
+#
+
+for a in "$@" ; do
+	OPTION_COUNT+=1
+	case "$a" in
+		--*=*)
+			[[ $a =~ --([^=]+)=(.*) ]]
+			l=${BASH_REMATCH[1]//-/_}
+			r=${BASH_REMATCH[2]}
+			eval ${l^^}=\"$r\"
+			;;
+		--*)
+			[[ $a =~ --(.+) ]]
+			l=${BASH_REMATCH[1]//-/_}
+			eval ${l^^}=1
+			;;
+	esac
+done
+
+runner() {
+	( set -x ; "$@" )
+}
+

tests/CI/installAsterisk.sh

+#!/bin/bash
+MAKE=`which make`
+if [ x"${@}" != x ] ; then
+	mkdir -p "${@}"
+fi
+destdir=${@:+DESTDIR=${@}}
+
+${MAKE} ${destdir} install || ${MAKE} ${destdir} NOISY_BUILD=yes install || exit 1
+${MAKE} ${destdir} samples
+if [ -n "${@}" ] ; then
+	sed -i -r -e "s@\[directories\]\(!\)@[directories]@g" $@/etc/asterisk/asterisk.conf
+	sed -i -r -e "s@ /(var|etc|usr)/@ ${@}/\1/@g" $@/etc/asterisk/asterisk.conf
+fi
+
+set +e
+chown -R jenkins:users ${@}/var/lib/asterisk
+chown -R jenkins:users ${@}/var/spool/asterisk
+chown -R jenkins:users ${@}/var/log/asterisk
+chown -R jenkins:users ${@}/var/run/asterisk
+chown -R jenkins:users ${@}/etc/asterisk
+[ ! -d ${@}/tmp/asterisk-jenkins ] && mkdir ${@}/tmp/asterisk-jenkins
+chown -R jenkins:users ${@}/tmp/asterisk-jenkins
+ldconfig
\ No newline at end of file

tests/CI/runTestsuite.sh

+#!/usr/bin/env bash
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+ASTETCDIR=$DESTDIR/etc/asterisk
+
+pushd $TESTSUITE_DIR
+
+runner sudo PYTHONPATH=./lib/python/ ./runtests.py --cleanup ${TEST_COMMAND} || :
+
+if [ -f asterisk-test-suite-report.xml ]  ; then
+	sudo chown jenkins:users asterisk-test-suite-report.xml
+fi
+
+runner ${CIDIR}/fixTestResults.py asterisk-test-suite-report.xml asterisk-test-suite-report.xml
+
+if [ -f core* ] ; then
+	echo "*** Found a core file after running unit tests ***"
+	sudo /var/lib/asterisk/scripts/ast_coredumper --no-default-search core*
+	exit 1
+fi
+
+popd
\ No newline at end of file

tests/CI/runUnittests.sh

+#!/usr/bin/env bash
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+ASTETCDIR=$DESTDIR/etc/asterisk
+
+echo "full => notice,warning,error,debug,verbose" > 	"$ASTETCDIR/logger.conf"
+
+echo "[default]" > "$ASTETCDIR/extensions.conf"
+
+cat <<-EOF > "$ASTETCDIR/manager.conf"
+	[general]
+	enabled=yes
+	bindaddr=127.0.0.1
+	port=5038
+
+	[test]
+	secret=test
+	read = system,call,log,verbose,agent,user,config,dtmf,reporting,cdr,dialplan
+	write = system,call,agent,user,config,command,reporting,originate
+EOF
+
+cat <<-EOF > "$ASTETCDIR/http.conf"
+	[general]
+	enabled=yes
+	bindaddr=127.0.0.1
+	port=8088
+EOF
+
+cat <<-EOF > "$ASTETCDIR/modules.conf"
+	[modules]
+	autoload=yes
+	noload=res_mwi_external.so
+	noload=res_mwi_external_ami.so
+	noload=res_ari_mailboxes.so
+	noload=res_stasis_mailbox.so
+EOF
+
+cat <<-EOF >> "$ASTETCDIR/sorcery.conf"
+	[res_pjsip_pubsub]
+	resource_list=memory
+EOF
+
+ASTERISK="$DESTDIR/usr/sbin/asterisk"
+CONFFILE=$ASTETCDIR/asterisk.conf
+OUTPUTDIR=${OUTPUT_DIR:-tests/CI/output/}
+OUTPUTFILE=${OUTPUT_XML:-${OUTPUTDIR}/unittests-results.xml}
+
+[ ! -d ${OUTPUTDIR} ] && mkdir -p $OUTPUTDIR
+sudo chown -R jenkins:users $OUTPUTDIR
+
+rm -rf $ASTETCDIR/extensions.{ael,lua} || :
+
+runner sudo $ASTERISK -U jenkins -G users -gn -C $CONFFILE
+sleep 3
+runner $ASTERISK -rx "core waitfullybooted" -C $CONFFILE
+sleep 1
+runner $ASTERISK -rx "${TEST_COMMAND:-test execute all}" -C $CONFFILE
+runner $ASTERISK -rx "test show results failed" -C $CONFFILE
+runner $ASTERISK -rx "test generate results xml $OUTPUTFILE" -C $CONFFILE
+runner $ASTERISK -rx "core stop now" -C $CONFFILE
+
+runner rsync -vaH $DESTDIR/var/log/asterisk/. $OUTPUTDIR
+
+sudo chown -R jenkins:users $OUTPUTDIR
+if [ -f core* ] ; then
+	echo "*** Found a core file after running unit tests ***"
+	$DESTDIR/var/lib/asterisk/scripts/ast_coredumper --no-default-search core*
+	exit 1
+fi

tests/CI/setupEnvironment.sh

+#!/usr/bin/env bash
+
+chmod 0750 /etc/sudoers.d
+chmod 0440 /etc/sudoers.d/jenkins
+
+chown root:root -R /root
+chmod -R go-rwx /root/.ssh
+chown -R jenkins:jenkins /home/jenkins
+chown -R jenkins:jenkins /srv/cache
+chown -R jenkins:jenkins /srv/jenkins
+chown -R jenkins:jenkins /srv/git
+chmod -R go-rwx /home/jenkins/.ssh
+chmod -R go-rwx /home/jenkins/.ssh/authorized_keys

tests/CI/unittests.jenkinsfile

+/*
+ * This pipeline is the "template" for the Asterisk Unit Tests multi-branch
+ * parent job.  Jenkins will automatically scan the branches in the "asterisk"
+ * or "Security-asterisk" projects in Gerrit and automatically create a branch-
+ * specific job for each branch it finds this file in.
+ *
+ * This file starts as a declarative pipeline because with a declarative
+ * pipeline, you can define the trigger in the pipeline file.  This keeps
+ * everything in one place.  We transition to scripted pipeline later on because
+ * we need to dynamically determine which docker image we're going to use and
+ * you can't do that in a delcarative pipeline.
+ */
+pipeline {
+	triggers {
+		/*
+		 * This trigger will match either the "asterisk" or "Security-asterisk"
+		 * projects.  The branch is taken from the branch this job was created
+		 * for.
+		 */
+		gerrit customUrl: '',
+			commentTextParameterMode: 'PLAIN',
+			commitMessageParameterMode: 'PLAIN',
+			gerritBuildStartedVerifiedValue: 0,
+			gerritBuildNotBuiltVerifiedValue: 0,
+			gerritBuildSuccessfulVerifiedValue: 1,
+			gerritBuildFailedVerifiedValue: -1,
+			gerritBuildUnstableVerifiedValue: -1,
+			gerritProjects: [
+				[branches: [[compareType: 'PLAIN', pattern: "${BRANCH_NAME}"]],
+					compareType: 'REG_EXP',
+					disableStrictForbiddenFileVerification: false,
+					pattern: '^(Security-)?asterisk.*'
+				]
+			],
+			silentMode: false,
+			triggerOnEvents: [
+				commentAddedContains('^recheck$'),
+				patchsetCreated(excludeDrafts: false,
+								excludeNoCodeChange: true,
+								excludeTrivialRebase: false),
+				draftPublished()
+			],
+			skipVote: [
+				onFailed: false,
+				onNotBuilt: true,
+				onSuccessful: false,
+				onUnstable: false
+			]
+	}
+	agent {
+		/* All of the stages need to be performed on a docker host */
+		label "swdev-docker"
+	}
+
+	stages {
+		stage ("Unit Tests-->") {
+			/*
+			 * Jenkins will try to automatically rebuild this job when
+			 * the jenkinsfile changes but since this job is dependent on
+			 * Gerrit, we really don't want to do anything in that case.
+			 */
+			when {
+				not { environment name: 'GERRIT_CHANGE_NUMBER', value: '' }
+			}
+			steps {
+				script {
+					stage ("Checkout") {
+						/*
+						 * Jenkins has already automatically checked out the base branch
+						 * for this change but we now need to check out the change itself
+						 * and rebase it on the current base branch.  If the rebase fails,
+						 * that's an indication to the user that they'll need to sort their
+						 * change out.
+						 *
+						 * The Gerrit Trigger provides all the URLs and refspecs to
+						 * check out the change.
+						 */
+						checkout scm: [$class: 'GitSCM',
+							branches: [[name: env.GERRIT_BRANCH ]],
+							extensions: [
+								[$class: 'CleanBeforeCheckout'],
+								[$class: 'PreBuildMerge', options: [
+									mergeRemote: env.GERRIT_NAME,
+									mergeTarget: env.GERRIT_BRANCH ]
+								],
+								[$class: 'CloneOption',
+									noTags: true,
+									depth: 10,
+									honorRefspec: true,
+									shallow: true
+								],
+								[$class: 'BuildChooserSetting',
+									buildChooser: [$class: 'GerritTriggerBuildChooser']
+								]
+							],
+							userRemoteConfigs: [
+								[name: env.GERRIT_NAME, refspec: env.GERRIT_REFSPEC, url: env.GIT_URL ]
+							]
+						]
+					}
+
+					def images = env.DOCKER_IMAGES.split(' ')
+					def r = currentBuild.startTimeInMillis % images.length
+					def ri = images[(int)r]
+					def randomImage = env.DOCKER_REGISTRY + "/" + ri;
+					def dockerOptions =
+						"-v /srv/jenkins:/srv/jenkins:rw -v /srv/cache:/srv/cache:rw" +
+						 " --entrypoint='' --name ${BUILD_TAG}-build"
+
+					docker.image(randomImage).inside(dockerOptions) {
+
+						stage ('Build') {
+							echo 'Building..'
+							sh './tests/CI/buildAsterisk.sh'
+
+							archiveArtifacts allowEmptyArchive: true, defaultExcludes: false, fingerprint: false,
+								artifacts: "tests/CI/output/*"
+						}
+
+						stage ('Test') {
+							def outputdir = "tests/CI/output/UnitTests"
+							def outputfile = "${outputdir}/unittests-results.xml"
+							def testcmd = "test execute all"
+
+							sh 'sudo ./tests/CI/installAsterisk.sh'
+							echo "tests/CI/runUnittests.sh --output-dir='${outputdir}' --output-xml='${outputfile}' --test-command='${testcmd}'"
+							sh "tests/CI/runUnittests.sh --output-dir='${outputdir}' --output-xml='${outputfile}' --test-command='${testcmd}'"
+
+							archiveArtifacts allowEmptyArchive: true, defaultExcludes: false, fingerprint: true,
+								artifacts: "${outputdir}/**"
+
+							junit testResults: outputfile,
+								healthScaleFactor: 1.0,
+								keepLongStdio: true
+						}
+					}
+
+					stage ('Cleanup') {
+						sh "sudo make distclean || : "
+					}
+				}
+			}
+		}
+	}
+	/*
+	 * The Gerrit Trigger will automatically post the "Verified" results back
+	 * to Gerrit but the verification publisher publishes extra stuff in the
+	 * "Code Review" section of the review.
+	 */
+	post {
+		success {
+			gerritverificationpublisher verifyStatusValue: 1, verifyStatusCategory: 'Passed',
+				verifyStatusURL: '', verifyStatusComment: '',
+				verifyStatusName: '', verifyStatusReporter: '',
+				verifyStatusRerun: 'recheck'
+		}
+		failure {
+			gerritverificationpublisher verifyStatusValue: -1, verifyStatusCategory: 'Failed',
+				verifyStatusURL: '', verifyStatusComment: '',
+				verifyStatusName: '', verifyStatusReporter: '',
+				verifyStatusRerun: 'recheck'
+		}
+		unstable {
+			gerritverificationpublisher verifyStatusValue: -1, verifyStatusCategory: 'Failed',
+				verifyStatusURL: '', verifyStatusComment: '',
+				verifyStatusName: '', verifyStatusReporter: '',
+				verifyStatusRerun: 'recheck'
+		}
+	}
+}