From b302ee6bd5899f21869dd594d78948e5de99f31c Mon Sep 17 00:00:00 2001
From: George Joseph <gjoseph@digium.com>
Date: Wed, 11 Jul 2018 05:14:49 -0600
Subject: [PATCH] CI: Initial commit for moving CI into source repo
Create tests/CI directory and add files used by Jenkins to
build and test Asterisk.
With this commit, Jenkins will run the Asterisk Unit Tests using
the Jenkinsfile at tests/CI/unittests.jenkinsfile. Bash scripts
to do the actual building and testing are also in the same directory.
Output is placed in tests/CI/output so that directory has been
added to .gitignore.
Change-Id: I9448065465e6de2b878634510ace8fd1ef378608
---
.gitignore | 2 +
tests/CI/buildAsterisk.sh | 103 ++++++++++++++++++++
tests/CI/ci.functions | 26 +++++
tests/CI/installAsterisk.sh | 23 +++++
tests/CI/runTestsuite.sh | 22 +++++
tests/CI/runUnittests.sh | 69 +++++++++++++
tests/CI/setupEnvironment.sh | 13 +++
tests/CI/unittests.jenkinsfile | 170 +++++++++++++++++++++++++++++++++
8 files changed, 428 insertions(+)
create mode 100755 tests/CI/buildAsterisk.sh
create mode 100644 tests/CI/ci.functions
create mode 100755 tests/CI/installAsterisk.sh
create mode 100755 tests/CI/runTestsuite.sh
create mode 100755 tests/CI/runUnittests.sh
create mode 100755 tests/CI/setupEnvironment.sh
create mode 100644 tests/CI/unittests.jenkinsfile
diff --git a/.gitignore b/.gitignore
index 3e09738e0e..0f31820627 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,5 @@ latex
doxygen.log
out/
*.orig
+tests/CI/output
+
diff --git a/tests/CI/buildAsterisk.sh b/tests/CI/buildAsterisk.sh
new file mode 100755
index 0000000000..fce81fcd81
--- /dev/null
+++ b/tests/CI/buildAsterisk.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+
+gen_cats() {
+ set +x
+ action=$1
+ shift
+ cats=$@
+
+ for x in $cats ; do
+ echo " --${action}-category ${x}"
+ done
+}
+
+gen_mods() {
+ set +x
+ action=$1
+ shift
+ mods=$@
+
+ for x in $mods ; do
+ echo " --${action} ${x}"
+ done
+}
+
+sudo mkdir -p /srv/cache/externals /srv/cache/sounds || :
+sudo chown -R jenkins:users /srv/cache
+[ ! -d tests/CI/output ] && mkdir tests/CI/output
+sudo chown -R jenkins:users tests/CI/output
+
+MAKE=`which make`
+printenv | sort
+
+common_config_args="--sysconfdir=/etc --with-pjproject-bundled"
+common_config_args+=" --with-sounds-cache=/srv/cache/sounds --with-externals-cache=/srv/cache/externals"
+common_config_args+=" --enable-dev-mode"
+export WGET_EXTRA_ARGS="--quiet"
+
+runner ./configure ${common_config_args} CCACHE_DISABLE=1 >tests/CI/output/configure.txt
+
+runner ${MAKE} menuselect.makeopts
+
+runner menuselect/menuselect `gen_mods enable DONT_OPTIMIZE BETTER_BACKTRACES MALLOC_DEBUG DO_CRASH TEST_FRAMEWORK` menuselect.makeopts
+runner menuselect/menuselect `gen_mods disable COMPILE_DOUBLE BUILD_NATIVE` menuselect.makeopts
+
+cat_enables="MENUSELECT_BRIDGES MENUSELECT_CEL MENUSELECT_CDR"
+cat_enables+=" MENUSELECT_CHANNELS MENUSELECT_CODECS MENUSELECT_FORMATS MENUSELECT_FUNCS"
+cat_enables+=" MENUSELECT_PBX MENUSELECT_RES MENUSELECT_UTILS MENUSELECT_TESTS"
+runner menuselect/menuselect `gen_cats enable $cat_enables` menuselect.makeopts
+
+mod_disables="res_digium_phone chan_vpb"
+[ "$BRANCH_NAME" == "master" ] && mod_disables+=" codec_opus codec_silk codec_g729a codec_siren7 codec_siren14"
+runner menuselect/menuselect `gen_mods disable $mod_disables` menuselect.makeopts
+
+mod_enables="app_voicemail app_directory FILE_STORAGE"
+mod_enables+=" res_mwi_external res_ari_mailboxes res_mwi_external_ami res_stasis_mailbox"
+mod_enables+=" CORE-SOUNDS-EN-GSM MOH-OPSOUND-GSM EXTRA-SOUNDS-EN-GSM"
+runner menuselect/menuselect `gen_mods enable $mod_enables` menuselect.makeopts
+
+runner ${MAKE} -j8 || runner ${MAKE} -j1 NOISY_BUILD=yes
+
+ALEMBIC=$(which alembic 2>/dev/null || : )
+if [ x"$ALEMBIC" = x ] ; then
+ echo "Alembic not installed"
+ exit 1
+fi
+
+cd contrib/ast-db-manage
+find -name *.pyc -delete
+out=$(alembic -c config.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+ >&2 echo "Alembic branches were found for config"
+ >&2 echo $out
+ exit 1
+else
+ >&2 echo "Alembic for 'config' OK"
+fi
+
+out=$(alembic -c cdr.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+ >&2 echo "Alembic branches were found for cdr"
+ >&2 echo $out
+ exit 1
+else
+ >&2 echo "Alembic for 'cdr' OK"
+fi
+
+out=$(alembic -c voicemail.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+ >&2 echo "Alembic branches were found for voicemail"
+ >&2 echo $out
+ exit 1
+else
+ >&2 echo "Alembic for 'voicemail' OK"
+fi
+
+if [ -f "doc/core-en_US.xml" ] ; then
+ ${MAKE} validate-docs || ${MAKE} NOISY_BUILD=yes validate-docs
+fi
+
+
diff --git a/tests/CI/ci.functions b/tests/CI/ci.functions
new file mode 100644
index 0000000000..f3de16e6d7
--- /dev/null
+++ b/tests/CI/ci.functions
@@ -0,0 +1,26 @@
+#
+# This file contains useful Bash functions
+# and can be "source"d from the scripts.
+#
+
+for a in "$@" ; do
+ OPTION_COUNT+=1
+ case "$a" in
+ --*=*)
+ [[ $a =~ --([^=]+)=(.*) ]]
+ l=${BASH_REMATCH[1]//-/_}
+ r=${BASH_REMATCH[2]}
+ eval ${l^^}=\"$r\"
+ ;;
+ --*)
+ [[ $a =~ --(.+) ]]
+ l=${BASH_REMATCH[1]//-/_}
+ eval ${l^^}=1
+ ;;
+ esac
+done
+
+runner() {
+ ( set -x ; "$@" )
+}
+
diff --git a/tests/CI/installAsterisk.sh b/tests/CI/installAsterisk.sh
new file mode 100755
index 0000000000..b75730f7c5
--- /dev/null
+++ b/tests/CI/installAsterisk.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+MAKE=`which make`
+if [ x"${@}" != x ] ; then
+ mkdir -p "${@}"
+fi
+destdir=${@:+DESTDIR=${@}}
+
+${MAKE} ${destdir} install || ${MAKE} ${destdir} NOISY_BUILD=yes install || exit 1
+${MAKE} ${destdir} samples
+if [ -n "${@}" ] ; then
+ sed -i -r -e "s@\[directories\]\(!\)@[directories]@g" $@/etc/asterisk/asterisk.conf
+ sed -i -r -e "s@ /(var|etc|usr)/@ ${@}/\1/@g" $@/etc/asterisk/asterisk.conf
+fi
+
+set +e
+chown -R jenkins:users ${@}/var/lib/asterisk
+chown -R jenkins:users ${@}/var/spool/asterisk
+chown -R jenkins:users ${@}/var/log/asterisk
+chown -R jenkins:users ${@}/var/run/asterisk
+chown -R jenkins:users ${@}/etc/asterisk
+[ ! -d ${@}/tmp/asterisk-jenkins ] && mkdir ${@}/tmp/asterisk-jenkins
+chown -R jenkins:users ${@}/tmp/asterisk-jenkins
+ldconfig
\ No newline at end of file
diff --git a/tests/CI/runTestsuite.sh b/tests/CI/runTestsuite.sh
new file mode 100755
index 0000000000..c96b9a453f
--- /dev/null
+++ b/tests/CI/runTestsuite.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+ASTETCDIR=$DESTDIR/etc/asterisk
+
+pushd $TESTSUITE_DIR
+
+runner sudo PYTHONPATH=./lib/python/ ./runtests.py --cleanup ${TEST_COMMAND} || :
+
+if [ -f asterisk-test-suite-report.xml ] ; then
+ sudo chown jenkins:users asterisk-test-suite-report.xml
+fi
+
+runner ${CIDIR}/fixTestResults.py asterisk-test-suite-report.xml asterisk-test-suite-report.xml
+
+if [ -f core* ] ; then
+ echo "*** Found a core file after running unit tests ***"
+ sudo /var/lib/asterisk/scripts/ast_coredumper --no-default-search core*
+ exit 1
+fi
+
+popd
\ No newline at end of file
diff --git a/tests/CI/runUnittests.sh b/tests/CI/runUnittests.sh
new file mode 100755
index 0000000000..a463e159df
--- /dev/null
+++ b/tests/CI/runUnittests.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+ASTETCDIR=$DESTDIR/etc/asterisk
+
+echo "full => notice,warning,error,debug,verbose" > "$ASTETCDIR/logger.conf"
+
+echo "[default]" > "$ASTETCDIR/extensions.conf"
+
+cat <<-EOF > "$ASTETCDIR/manager.conf"
+ [general]
+ enabled=yes
+ bindaddr=127.0.0.1
+ port=5038
+
+ [test]
+ secret=test
+ read = system,call,log,verbose,agent,user,config,dtmf,reporting,cdr,dialplan
+ write = system,call,agent,user,config,command,reporting,originate
+EOF
+
+cat <<-EOF > "$ASTETCDIR/http.conf"
+ [general]
+ enabled=yes
+ bindaddr=127.0.0.1
+ port=8088
+EOF
+
+cat <<-EOF > "$ASTETCDIR/modules.conf"
+ [modules]
+ autoload=yes
+ noload=res_mwi_external.so
+ noload=res_mwi_external_ami.so
+ noload=res_ari_mailboxes.so
+ noload=res_stasis_mailbox.so
+EOF
+
+cat <<-EOF >> "$ASTETCDIR/sorcery.conf"
+ [res_pjsip_pubsub]
+ resource_list=memory
+EOF
+
+ASTERISK="$DESTDIR/usr/sbin/asterisk"
+CONFFILE=$ASTETCDIR/asterisk.conf
+OUTPUTDIR=${OUTPUT_DIR:-tests/CI/output/}
+OUTPUTFILE=${OUTPUT_XML:-${OUTPUTDIR}/unittests-results.xml}
+
+[ ! -d ${OUTPUTDIR} ] && mkdir -p $OUTPUTDIR
+sudo chown -R jenkins:users $OUTPUTDIR
+
+rm -rf $ASTETCDIR/extensions.{ael,lua} || :
+
+runner sudo $ASTERISK -U jenkins -G users -gn -C $CONFFILE
+sleep 3
+runner $ASTERISK -rx "core waitfullybooted" -C $CONFFILE
+sleep 1
+runner $ASTERISK -rx "${TEST_COMMAND:-test execute all}" -C $CONFFILE
+runner $ASTERISK -rx "test show results failed" -C $CONFFILE
+runner $ASTERISK -rx "test generate results xml $OUTPUTFILE" -C $CONFFILE
+runner $ASTERISK -rx "core stop now" -C $CONFFILE
+
+runner rsync -vaH $DESTDIR/var/log/asterisk/. $OUTPUTDIR
+
+sudo chown -R jenkins:users $OUTPUTDIR
+if [ -f core* ] ; then
+ echo "*** Found a core file after running unit tests ***"
+ $DESTDIR/var/lib/asterisk/scripts/ast_coredumper --no-default-search core*
+ exit 1
+fi
diff --git a/tests/CI/setupEnvironment.sh b/tests/CI/setupEnvironment.sh
new file mode 100755
index 0000000000..eef47a65ce
--- /dev/null
+++ b/tests/CI/setupEnvironment.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+chmod 0750 /etc/sudoers.d
+chmod 0440 /etc/sudoers.d/jenkins
+
+chown root:root -R /root
+chmod -R go-rwx /root/.ssh
+chown -R jenkins:jenkins /home/jenkins
+chown -R jenkins:jenkins /srv/cache
+chown -R jenkins:jenkins /srv/jenkins
+chown -R jenkins:jenkins /srv/git
+chmod -R go-rwx /home/jenkins/.ssh
+chmod -R go-rwx /home/jenkins/.ssh/authorized_keys
diff --git a/tests/CI/unittests.jenkinsfile b/tests/CI/unittests.jenkinsfile
new file mode 100644
index 0000000000..0925910368
--- /dev/null
+++ b/tests/CI/unittests.jenkinsfile
@@ -0,0 +1,170 @@
+/*
+ * This pipeline is the "template" for the Asterisk Unit Tests multi-branch
+ * parent job. Jenkins will automatically scan the branches in the "asterisk"
+ * or "Security-asterisk" projects in Gerrit and automatically create a branch-
+ * specific job for each branch it finds this file in.
+ *
+ * This file starts as a declarative pipeline because with a declarative
+ * pipeline, you can define the trigger in the pipeline file. This keeps
+ * everything in one place. We transition to scripted pipeline later on because
+ * we need to dynamically determine which docker image we're going to use and
+ * you can't do that in a delcarative pipeline.
+ */
+pipeline {
+ triggers {
+ /*
+ * This trigger will match either the "asterisk" or "Security-asterisk"
+ * projects. The branch is taken from the branch this job was created
+ * for.
+ */
+ gerrit customUrl: '',
+ commentTextParameterMode: 'PLAIN',
+ commitMessageParameterMode: 'PLAIN',
+ gerritBuildStartedVerifiedValue: 0,
+ gerritBuildNotBuiltVerifiedValue: 0,
+ gerritBuildSuccessfulVerifiedValue: 1,
+ gerritBuildFailedVerifiedValue: -1,
+ gerritBuildUnstableVerifiedValue: -1,
+ gerritProjects: [
+ [branches: [[compareType: 'PLAIN', pattern: "${BRANCH_NAME}"]],
+ compareType: 'REG_EXP',
+ disableStrictForbiddenFileVerification: false,
+ pattern: '^(Security-)?asterisk.*'
+ ]
+ ],
+ silentMode: false,
+ triggerOnEvents: [
+ commentAddedContains('^recheck$'),
+ patchsetCreated(excludeDrafts: false,
+ excludeNoCodeChange: true,
+ excludeTrivialRebase: false),
+ draftPublished()
+ ],
+ skipVote: [
+ onFailed: false,
+ onNotBuilt: true,
+ onSuccessful: false,
+ onUnstable: false
+ ]
+ }
+ agent {
+ /* All of the stages need to be performed on a docker host */
+ label "swdev-docker"
+ }
+
+ stages {
+ stage ("Unit Tests-->") {
+ /*
+ * Jenkins will try to automatically rebuild this job when
+ * the jenkinsfile changes but since this job is dependent on
+ * Gerrit, we really don't want to do anything in that case.
+ */
+ when {
+ not { environment name: 'GERRIT_CHANGE_NUMBER', value: '' }
+ }
+ steps {
+ script {
+ stage ("Checkout") {
+ /*
+ * Jenkins has already automatically checked out the base branch
+ * for this change but we now need to check out the change itself
+ * and rebase it on the current base branch. If the rebase fails,
+ * that's an indication to the user that they'll need to sort their
+ * change out.
+ *
+ * The Gerrit Trigger provides all the URLs and refspecs to
+ * check out the change.
+ */
+ checkout scm: [$class: 'GitSCM',
+ branches: [[name: env.GERRIT_BRANCH ]],
+ extensions: [
+ [$class: 'CleanBeforeCheckout'],
+ [$class: 'PreBuildMerge', options: [
+ mergeRemote: env.GERRIT_NAME,
+ mergeTarget: env.GERRIT_BRANCH ]
+ ],
+ [$class: 'CloneOption',
+ noTags: true,
+ depth: 10,
+ honorRefspec: true,
+ shallow: true
+ ],
+ [$class: 'BuildChooserSetting',
+ buildChooser: [$class: 'GerritTriggerBuildChooser']
+ ]
+ ],
+ userRemoteConfigs: [
+ [name: env.GERRIT_NAME, refspec: env.GERRIT_REFSPEC, url: env.GIT_URL ]
+ ]
+ ]
+ }
+
+ def images = env.DOCKER_IMAGES.split(' ')
+ def r = currentBuild.startTimeInMillis % images.length
+ def ri = images[(int)r]
+ def randomImage = env.DOCKER_REGISTRY + "/" + ri;
+ def dockerOptions =
+ "-v /srv/jenkins:/srv/jenkins:rw -v /srv/cache:/srv/cache:rw" +
+ " --entrypoint='' --name ${BUILD_TAG}-build"
+
+ docker.image(randomImage).inside(dockerOptions) {
+
+ stage ('Build') {
+ echo 'Building..'
+ sh './tests/CI/buildAsterisk.sh'
+
+ archiveArtifacts allowEmptyArchive: true, defaultExcludes: false, fingerprint: false,
+ artifacts: "tests/CI/output/*"
+ }
+
+ stage ('Test') {
+ def outputdir = "tests/CI/output/UnitTests"
+ def outputfile = "${outputdir}/unittests-results.xml"
+ def testcmd = "test execute all"
+
+ sh 'sudo ./tests/CI/installAsterisk.sh'
+ echo "tests/CI/runUnittests.sh --output-dir='${outputdir}' --output-xml='${outputfile}' --test-command='${testcmd}'"
+ sh "tests/CI/runUnittests.sh --output-dir='${outputdir}' --output-xml='${outputfile}' --test-command='${testcmd}'"
+
+ archiveArtifacts allowEmptyArchive: true, defaultExcludes: false, fingerprint: true,
+ artifacts: "${outputdir}/**"
+
+ junit testResults: outputfile,
+ healthScaleFactor: 1.0,
+ keepLongStdio: true
+ }
+ }
+
+ stage ('Cleanup') {
+ sh "sudo make distclean || : "
+ }
+ }
+ }
+ }
+ }
+ /*
+ * The Gerrit Trigger will automatically post the "Verified" results back
+ * to Gerrit but the verification publisher publishes extra stuff in the
+ * "Code Review" section of the review.
+ */
+ post {
+ success {
+ gerritverificationpublisher verifyStatusValue: 1, verifyStatusCategory: 'Passed',
+ verifyStatusURL: '', verifyStatusComment: '',
+ verifyStatusName: '', verifyStatusReporter: '',
+ verifyStatusRerun: 'recheck'
+ }
+ failure {
+ gerritverificationpublisher verifyStatusValue: -1, verifyStatusCategory: 'Failed',
+ verifyStatusURL: '', verifyStatusComment: '',
+ verifyStatusName: '', verifyStatusReporter: '',
+ verifyStatusRerun: 'recheck'
+ }
+ unstable {
+ gerritverificationpublisher verifyStatusValue: -1, verifyStatusCategory: 'Failed',
+ verifyStatusURL: '', verifyStatusComment: '',
+ verifyStatusName: '', verifyStatusReporter: '',
+ verifyStatusRerun: 'recheck'
+ }
+ }
+}
--
GitLab