ci: add documntation checking and posting to AWS S3

Signed-off-by: Anas Nashif <anas.nashif@intel.com>

.shippable.yml

@@ -10,12 +10,9 @@ env:
         - ZEPHYR_SDK_INSTALL_DIR=/opt/sdk/zephyr-sdk-0.9
         - ZEPHYR_GCC_VARIANT=zephyr
         - USE_CCACHE=1
-        - secure: CaE0YOxMfS71yTJsLOUMAXyvrOfgPbT6NLakwXShPHFF+aqqu9UyrmwFE1UfNxDrFOa3h0gxmbMRJAdGPLdKeLmGlLiL96XMhpaZIWYmAD2/Kfx9wb+1zfYISrh9k11QIifbB5JpeiFzNrrwYLOv5Gqn2fkAgvSe0BEKoh6weCvMXHgxwJR/I5gtQYwZXI6arvOTWlVgRpXeqURcJbthsmp7/Bc4MctgiRXmBxeyvi+OTVe1u/sNPVf51ZYcNdaqw+xRp9xFeg09EP87QPlDHV+g9dPWuGvGHAwQ86TD8hkpjurLO3O8GHCXena7Ft0/t9iL4RBecUIBplISNuaK6Q==
     matrix:
-        - ARCH="-a x86 -a riscv32 -a nios2" RUN_COMPLIANCE="1"
+        - ARCH="-a x86 -a riscv32" RUN_COMPLIANCE="1"
-        - ARCH="-a arm -a arc"
+        - ARCH="-a arm -a arc -a nios2"
-        #- ARCH="-a x86" RUN_COMPLIANCE="1"
-        #- ARCH="-a arc -a riscv32 -a nios2"
 
 build:
     cache: true
@@ -28,7 +25,6 @@ build:
         options: "-e HOME=/home/buildslave --privileged=true --tty --net=bridge --user buildslave"
 
     ci:
-      - env
       - export CCACHE_DIR=${SHIPPABLE_BUILD_DIR}/ccache/.ccache
       - export COMMIT_RANGE=${SHIPPABLE_COMMIT_RANGE}
       - source zephyr-env.sh
@@ -36,24 +32,42 @@ build:
       - make host-tools
       - export PREBUILT_HOST_TOOLS=${ZEPHYR_BASE}/bin
       - >
-          if [ "$RUN_COMPLIANCE" = "1" -a "$IS_PULL_REQUEST" = "true"]; then
+          if [ "$IS_PULL_REQUEST" = "true" ]; then
+            S3_PATH="s3://zephyr-logs/pull-requests/${PULL_REQUEST}"
+          else
+            if [ "$JOB_TRIGGERED_BY_NAME" = "undefined" ]; then
+              LOG_TYPE="manual";
+            else
+              LOG_TYPE=${JOB_TRIGGERED_BY_NAME};
+            fi;
+
+            S3_PATH="s3://zephyr-logs/${LOG_TYPE}/${BUILD_NUMBER}";
+          fi;
+      - >
+          if [ "$RUN_COMPLIANCE" = "1" -a "$IS_PULL_REQUEST" = "true" ]; then
             echo "Building a Pull Request";
-            errors=$(./scripts/ci/check-compliance.py);
+            echo "- Building Documentation";
-            cat compliance.xml
+            make htmldocs > doc.log 2>&1;
+            ./scripts/filter-known-issues.py --config-dir .known-issues/doc/ doc.log > doc.warnings;
+            if [ -s doc.warnings ]; then
+              echo " => New documentation warnings/errors";
+            fi;
+            echo "- Verify commit message and coding style";
+            ./scripts/ci/check-compliance.py || true;
           fi;
       - >
           if [ "$JOB_TRIGGERED_BY_NAME" = "daily-verify-asserts" ]; then
-            echo "Building with --all --enable-slow -R";
+            echo "- Building with --all --enable-slow -R";
             COVERAGE="--all --enable-slow -R";
           fi;
       - >
           if [ "$JOB_TRIGGERED_BY_NAME" = "daily-verify" ]; then
-            echo "Building with --all --enable-slow";
+            echo "- Building with --all --enable-slow";
             COVERAGE="--all --enable-slow";
           fi;
       - >
           if [ "$JOB_TRIGGERED_BY_NAME" = "code-scan" ]; then
-            echo "Building basic sanitycheck";
+            echo "- Building basic sanitycheck";
             wget https://scan.coverity.com/download/linux64 --post-data "token=${COVERITY_TOKEN}&project=Zephyr" -O coverity_tool.tgz;
             tar xvf coverity_tool.tgz;
             rm -f coverity_tool.tgz;
@@ -65,23 +79,44 @@ build:
             ./scripts/sanitycheck ${PLATFORMS} ${ARCH} ${COVERAGE} ${SANITYCHECK_OPTIONS} || ./scripts/sanitycheck ${PLATFORMS} ${ARCH} ${COVERAGE} ${SANITYCHECK_OPTIONS_RETRY};
           fi
       - ccache -s
-    post_ci:
+    on_success:
       - rm -rf sanity-out out-2nd-pass
       - mkdir -p shippable/testresults
       - >
           if [ -e compliance.xml ]; then
             cp compliance.xml shippable/testresults/;
+            aws s3 cp compliance.xml ${S3_PATH}/;
           fi;
-    on_failure:
       - >
           if [ -e ./scripts/sanity_chk/last_sanity.xml ]; then
             cp ./scripts/sanity_chk/last_sanity.xml shippable/testresults/;
+            aws s3 cp ./scripts/sanity_chk/last_sanity.xml ${S3_PATH}/sanitycheck.xml;
+          fi;
+    on_failure:
+      - rm -rf sanity-out out-2nd-pass
+      - mkdir -p shippable/testresults
+      - >
+          if [ -e compliance.xml ]; then
+            cp compliance.xml shippable/testresults/;
+            aws s3 cp compliance.xml ${S3_PATH}/;
+          fi;
+      - >
+          if [ -e ./scripts/sanity_chk/last_sanity.xml ]; then
+            cp ./scripts/sanity_chk/last_sanity.xml shippable/testresults/;
+            aws s3 cp ./scripts/sanity_chk/last_sanity.xml ${S3_PATH}/sanitycheck.xml;
           fi;
-    on_success:
-      - cp ./scripts/sanity_chk/last_sanity.xml shippable/testresults/
 
 integrations:
   notifications:
+    - integrationName: slack_integration
+      type: slack
+      recipients:
+        - "#ci"
+      branches:
+        only:
+          - master
+      on_success: never
+      on_failure: always
     - integrationName: email
       type: email
       recipients:

scripts/ci/build-docs.sh

@@ -1,20 +1,19 @@
 #!/bin/bash
 set -e
 
-sudo pip install pygithub
-echo "- Checkpatch"
-cd ${ZEPHYRREPO_STATE}
-source zephyr-env.sh
-git diff ${ZEPHYR_CIREPO_VERSIONNAME} | ${ZEPHYR_BASE}/scripts/checkpatch.pl --mailback --no-tree
-
 echo "- Install dependencies"
 sudo apt-get install doxygen make
-sudo pip install breathe sphinx
+sudo pip install breathe sphinx awscli sphinx_rtd_theme
+
+cd ${TESTING_REPO_STATE}
+source zephyr-env.sh
+
+cp -a /build/IN/docs-theme-repo/gitRepo doc/themes/zephyr-docs-theme
+ls -la doc/themes
 
 echo "- Building docs..."
-make htmldocs > doc.log 2>&1
+make DOC_TAG=daily htmldocs > doc.log 2>&1
-echo "- Look for new warnings..."
+echo "- Uploading to AWS S3..."
-#./scripts/filter-known-issues.py --config-dir .known-issues/doc/ doc.log > doc.warnings
+aws s3 sync --quiet --delete doc/_build/html s3://zephyr-docs/online/dev
-#cat doc.warnings
-#test -s doc.warnings && exit 0 # FIXME
 
+echo "Done"

scripts/ci/check-compliance.py

@@ -5,6 +5,8 @@ import re
 import os
 import xml.etree.ElementTree as ET
 
+DOCS_WARNING_FILE = "doc.warnings"
+
 commit_range = os.environ['COMMIT_RANGE']
 cwd = os.environ['ZEPHYR_BASE']
 
@@ -37,17 +39,28 @@ def run_checkpatch(tc):
     except subprocess.CalledProcessError as ex:
         m = re.search("([1-9][0-9]*) errors,", str(ex.output))
         if m:
-            failure = ET.SubElement(tc, 'failure', type="failure", message="check patch issues")
+            failure = ET.SubElement(tc, 'failure', type="failure", message="checkpatch issues")
             failure.text = (str(ex.output))
             return 1
 
     return 0
 
 
+def check_doc(tc):
+
+    if os.path.exists(DOCS_WARNING_FILE) and os.path.getsize(DOCS_WARNING_FILE) > 0:
+        with open(DOCS_WARNING_FILE, "r") as f:
+            log = f.read()
+            failure = ET.SubElement(tc, 'failure', type="failure",
+                        message="documentation issues")
+            failure.text = (str(log))
+        return 1
+
+    return 0
 
 
 
-tests = {"gitlint":run_gitlint, "checkpatch":run_checkpatch}
+tests = { "gitlint":run_gitlint, "checkpatch":run_checkpatch, "documentation":check_doc }
 
 def run_tests():
     run = "Commit"