ci: add CI/CD integration and related scripts

Add initial .shippable.yml for CI integration on github and related
scripts.

Change-Id: I095d125e780bba980e635e218205c8741e753a8e
Signed-off-by: Anas Nashif <anas.nashif@intel.com>

.shippable.yml

@@ -0,0 +1,92 @@
+language: c
+
+compiler: gcc
+
+env:
+    global:
+        - SDK=0.9
+        - SANITYCHECK_OPTIONS=" --inline-logs"
+        - SANITYCHECK_OPTIONS_RETRY=" --inline-logs --only-failed --outdir=out-2nd-pass"
+        - ZEPHYR_SDK_INSTALL_DIR=/opt/sdk/zephyr-sdk-0.9
+        - ZEPHYR_GCC_VARIANT=zephyr
+        - USE_CCACHE=1
+        - secure: CaE0YOxMfS71yTJsLOUMAXyvrOfgPbT6NLakwXShPHFF+aqqu9UyrmwFE1UfNxDrFOa3h0gxmbMRJAdGPLdKeLmGlLiL96XMhpaZIWYmAD2/Kfx9wb+1zfYISrh9k11QIifbB5JpeiFzNrrwYLOv5Gqn2fkAgvSe0BEKoh6weCvMXHgxwJR/I5gtQYwZXI6arvOTWlVgRpXeqURcJbthsmp7/Bc4MctgiRXmBxeyvi+OTVe1u/sNPVf51ZYcNdaqw+xRp9xFeg09EP87QPlDHV+g9dPWuGvGHAwQ86TD8hkpjurLO3O8GHCXena7Ft0/t9iL4RBecUIBplISNuaK6Q==
+    matrix:
+        - ARCH="" RUN_COMPLIANCE="1"
+        #- ARCH="-a x86" RUN_COMPLIANCE="1"
+        #- ARCH="-a arm"
+        #- ARCH="-a arc -a riscv32 -a nios2"
+
+build:
+    cache: true
+    cache_dir_list:
+        - ${SHIPPABLE_BUILD_DIR}/ccache
+    pre_ci_boot:
+        image_name: nashif/zephyr
+        image_tag: master.6
+        pull: true
+        options: "-e HOME=/home/buildslave --privileged=true --tty --net=bridge --user buildslave"
+
+    ci:
+      - env
+      - export CCACHE_DIR=${SHIPPABLE_BUILD_DIR}/ccache/.ccache
+      - export COMMIT_RANGE=${SHIPPABLE_COMMIT_RANGE}
+      - source zephyr-env.sh
+      - ccache -s --max-size=2000M
+      - make host-tools
+      - export PREBUILT_HOST_TOOLS=${ZEPHYR_BASE}/bin
+      - >
+          if [ "$RUN_COMPLIANCE" = "1" -a "$IS_PULL_REQUEST" = "true"]; then
+            errors=$(./scripts/ci/check-compliance.py);
+            cat compliance.xml
+          fi;
+      - >
+          if [ "$JOB_TRIGGERED_BY_NAME" = "daily-verify-asserts" ]; then
+            COVERAGE="--all --enable-slow -R";
+          fi;
+      - >
+          if [ "$JOB_TRIGGERED_BY_NAME" = "daily-verify" ]; then
+            COVERAGE="--all --enable-slow";
+          fi;
+      - >
+          if [ "$JOB_TRIGGERED_BY_NAME" = "code-scan" ]; then
+            wget https://scan.coverity.com/download/linux64 --post-data "token=${COVERITY_TOKEN}&project=Zephyr" -O coverity_tool.tgz;
+            tar xvf coverity_tool.tgz;
+            rm -f coverity_tool.tgz;
+            mv cov-*  cov-analysis;
+            ./scripts/ci/run-coverity.sh
+          fi;
+      - >
+          if [ "$JOB_TRIGGERED_BY_NAME" != "code-scan" ]; then
+            ./scripts/sanitycheck ${PLATFORMS} ${ARCH} ${COVERAGE} ${SANITYCHECK_OPTIONS} || ./scripts/sanitycheck ${PLATFORMS} ${ARCH} ${COVERAGE} ${SANITYCHECK_OPTIONS_RETRY};
+          fi
+      - ccache -s
+    post_ci:
+      - rm -rf sanity-out out-2nd-pass
+      - mkdir -p shippable/testresults
+      - >
+          if [ -e compliance.xml ]; then
+            cp compliance.xml shippable/testresults/;
+          fi;
+    on_failure:
+      - >
+          if [ -e ./scripts/sanity_chk/last_sanity.xml ]; then
+            cp ./scripts/sanity_chk/last_sanity.xml shippable/testresults/;
+          fi;
+    on_success:
+      - cp ./scripts/sanity_chk/last_sanity.xml shippable/testresults/
+
+integrations:
+  notifications:
+    - integrationName: email
+      type: email
+      recipients:
+        - builds@zephyrproject.org
+      branches:
+        only:
+          - master
+          - net
+          - bluetooth
+          - arm
+      on_success: never
+      on_failure: never

scripts/ci/build-docs.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+sudo pip install pygithub
+echo "- Checkpatch"
+cd ${ZEPHYRREPO_STATE}
+source zephyr-env.sh
+git diff ${ZEPHYR_CIREPO_VERSIONNAME} | ${ZEPHYR_BASE}/scripts/checkpatch.pl --mailback --no-tree
+
+echo "- Install dependencies"
+sudo apt-get install doxygen make
+sudo pip install breathe sphinx
+
+echo "- Building docs..."
+make htmldocs > doc.log 2>&1
+echo "- Look for new warnings..."
+#./scripts/filter-known-issues.py --config-dir .known-issues/doc/ doc.log > doc.warnings
+#cat doc.warnings
+#test -s doc.warnings && exit 0 # FIXME
+

scripts/ci/check-compliance.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+import sys
+import subprocess
+import re
+import os
+import xml.etree.ElementTree as ET
+
+commit_range = os.environ['COMMIT_RANGE']
+cwd = os.environ['ZEPHYR_BASE']
+
+def run_gitlint(tc):
+    proc = subprocess.Popen('gitlint --commits %s' %(commit_range),
+            cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    msg = ""
+    if proc.wait() != 0:
+        msg = proc.stdout.read()
+
+    if msg != "":
+        failure = ET.SubElement(tc, 'failure', type="failure", message="commit message error")
+        failure.text = (str(msg))
+        return 1
+
+    return 0
+
+
+def run_checkpatch(tc):
+    output = None
+    out = ""
+
+    diff = subprocess.Popen(('git', 'diff', '%s' %(commit_range)), stdout=subprocess.PIPE)
+    try:
+        output = subprocess.check_output(('%s/scripts/checkpatch.pl' %cwd,
+            '--mailback', '--no-tree', '-'), stdin=diff.stdout,
+            stderr=subprocess.STDOUT, shell=True)
+
+    except subprocess.CalledProcessError as ex:
+        m = re.search("([1-9][0-9]*) errors,", str(ex.output))
+        if m:
+            failure = ET.SubElement(tc, 'failure', type="failure", message="check patch issues")
+            failure.text = (str(ex.output))
+            return 1
+
+    return 0
+
+
+
+
+
+tests = {"gitlint":run_gitlint, "checkpatch":run_checkpatch}
+
+def run_tests():
+    run = "Commit"
+    eleTestsuite = None
+    fails = 0
+    passes = 0
+    errors = 0
+    total = 0
+    filename = "compliance.xml"
+
+    eleTestsuites = ET.Element('testsuites')
+    eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite', name=run,
+            tests="%d" %(errors + passes + fails),  failures="%d" %fails,  errors="%d" %errors, skip="0")
+
+    for test in tests.keys():
+
+        total += 1
+        eleTestcase = ET.SubElement(eleTestsuite, 'testcase', name="%s" %(test),
+                time="0")
+
+        fails += tests[test](eleTestcase)
+
+
+    eleTestsuite.set("tests", "%s" %total)
+    eleTestsuite.set("failures",  "%s" %fails)
+
+    result = ET.tostring(eleTestsuites)
+    f = open(filename, 'wb')
+    f.write(result)
+    f.close()
+    return fails
+
+fails = run_tests()
+print(fails)
+sys.exit(fails)

scripts/ci/run-coverity.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# This script builds the application using the Coverity Scan build tool,
+# and prepares the archive for uploading to the cloud static analyzer.
+#
+
+function die() { echo "$@" 1>&2; exit 1; }
+
+rm -rf /tmp/cov-build/cov-int
+export PATH=$PATH:${SHIPPABLE_BUILD_DIR}/cov-analysis/bin
+which cov-configure && which cov-build || die "Coverity Build Tool is not in PATH"
+
+#cov-configure --comptype gcc --compiler i586-zephyr-elfiamcu-gcc --template
+#cov-build --dir /tmp/cov-build/cov-int sanitycheck -a x86 --all -b
+
+#cov-configure --comptype gcc --compiler arm-zephyr-eabi-gcc --template
+#cov-build --dir /tmp/cov-build/cov-int sanitycheck -a arm --all -b
+
+cov-configure --comptype gcc --compiler arc-zephyr-elf-gcc --template
+cov-build --dir /tmp/cov-build/cov-int sanitycheck -a arc --all -b
+
+cd /tmp/cov-build
+ls -lR cov-int
+tar czvf coverity.tgz cov-int
+
+echo "Done. Please submit the archive to Coverity Scan now."