CI: sources check

Start working on custom code-style/code-requirements file checker.
So far we check only for missing license, for non-standard EOL (\r) and
for files that not-compliant with clang-format.

.gitlab-ci.yml

@@ -142,6 +142,11 @@ static-analysis:clang-tidy:
     - clang-tidy --version
     - jq ".[].file" build/compile_commands.json | tr -d "\"" | grep -v "test-src" | xargs -n1 -P$(grep -c processor /proc/cpuinfo) clang-tidy -p build/
+static-analysis:sources-check:
+  extends: .template:static-analysis
+  script:
+    - python3 extra/scripts/sources-check.py .
+
 #######################################################################################################################
 # notify

extra/scripts/sources-check.py

+#!/usr/bin/env python3
+
+import logging
+import io
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+
+LOGGER = logging.getLogger()
+LOGGER.setLevel(logging.DEBUG)
+handler = logging.StreamHandler(sys.stderr)
+handler.setLevel(logging.DEBUG)
+handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(levelname)8s: %(message)s'))
+LOGGER.addHandler(handler)
+
+
+class Recomendation:
+    def __init__(self, check, filepath, msg):
+        self.check = check
+        self.filepath = filepath
+        self.msg = msg
+
+    def __str__(self):
+        return "[%s] %s: %s" % (self.check, self.filepath, self.msg)
+
+class CheckResult:
+    def __init__(self, msg = None):
+        self.msg = msg
+
+    def success(self):
+        return self.msg is None
+
+
+class Check:
+    def __init__(self):
+        self.checks = list()
+
+    def register(self, foo):
+        self.checks.append(foo)
+
+    def run(self, rootdir):
+        ret = True
+
+        for root, _dirs, files in os.walk(rootdir):
+            for filename in files:
+                abs_filepath = os.path.join(root, filename)
+                rel_filepath = os.path.relpath(abs_filepath, rootdir)
+
+                if 'src' not in rel_filepath:
+                    continue
+                if not rel_filepath.startswith("alib2") and not rel_filepath.startswith("aql"):
+                    continue
+                if filename == "CMakeLists.txt":
+                    continue
+
+                with open(abs_filepath, 'rb') as fileobj:
+                    bin_contents = fileobj.read()
+
+                with open(abs_filepath, 'r') as fileobj:
+                    str_contents = fileobj.read()
+
+                for check in self.checks:
+                    for res in check(abs_filepath, rel_filepath, str_contents, bin_contents):
+                        if res.success() is False:
+                            print(Recomendation(check.__name__, rel_filepath, res.msg))
+                            ret = False
+
+        return ret
+
+
+def no_CR(abs_filepath, filepath, str_contents, bin_contents):
+    if "\r".encode() in bin_contents:
+        yield CheckResult("File contains a CR byte (\\r)")
+
+
+def clang_format(abs_filepath, filepath, str_contents, bin_contents):
+    proc = subprocess.run(["clang-format", "--dry-run", abs_filepath], capture_output=True)
+    if proc.stderr != b'':
+        yield CheckResult("File not properly formatted")
+
+
+COPYRIGHT = re.compile(r"""/\*
+ \* This file is part of Algorithms library toolkit\.
+( \* Copyright \(C\) \d+ (- \d+)?.* \(.*\)\n)*
+ \* Algorithms library toolkit is free software: you can redistribute it and/or modify
+ \* it under the terms of the GNU General Public License as published by
+ \* the Free Software Foundation, either version 3 of the License, or
+ \* \(at your option\) any later version\.
+
+ \* Algorithms library toolkit is distributed in the hope that it will be useful,
+ \* but WITHOUT ANY WARRANTY; without even the implied warranty of
+ \* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE\.  See the
+ \* GNU General Public License for more details\.
+
+ \* You should have received a copy of the GNU General Public License
+ \* along with Algorithms library toolkit\.  If not, see <http://www\.gnu\.org\/licenses/>\.
+ \*\/""", re.MULTILINE)
+
+COPYRIGHT_AUTHOR = re.compile(r"Copyright \(C\) (?P<year>\d+) (- (?P<year2>\d+))?(?P<author>.*) \((?P<email>.*)\)")
+
+def extract_git_history(abs_filepath):
+    proc = subprocess.run(["git", "--git-dir", os.path.join(REPO_PATH, ".git"), "log", '--pretty=format:"%an|%ae|%as"', abs_filepath], capture_output=True)
+
+    res = dict()
+    with io.StringIO(proc.stdout.decode('utf-8')) as f:
+        for line in f.readlines():
+            entry = line.strip().strip('"').split("|")
+            if entry[0] not in res:
+                res[entry[0]] = dict()
+            if 'emails' not in res[entry[0]]:
+                res[entry[0]]['emails'] = set()
+            if 'years' not in res[entry[0]]:
+                res[entry[0]]['years'] = set()
+            res[entry[0]]['emails'].add(entry[1].lower())
+            res[entry[0]]['years'].add(entry[2].split("-")[0])
+
+    return res
+
+def copyright(abs_filepath, filepath, str_contents, bin_contents):
+    m = COPYRIGHT.match(str_contents)
+    if not m:
+        yield CheckResult("Missing LICENSE preambule")
+        return
+
+    git_history = extract_git_history(abs_filepath)
+    copyright_history = dict()
+
+    for m in COPYRIGHT_AUTHOR.finditer(str_contents):
+        year_from = m.group('year')
+        year_to = m.group('year2') if m.group('year2') else year_from
+        author = m.group('author')
+        email = m.group('email').lower()
+
+        copyright_history[author] = {'email': email, 'year_range': (year_from, year_to)}
+
+    for author, values in git_history.items():
+        if author not in copyright_history:
+            yield CheckResult("Author '{}' listed in git history but not in copyright".format(author))
+            return
+
+        if not all(email.lower() == copyright_history[author]['email'].lower() for email in values['emails']):
+            yield CheckResult("Author '{}' has none of his git history emails ({}) in Copyright".format(author, values['emails']))
+
+    for author, values in copyright_history.items():
+        if author not in git_history:
+            yield CheckResult("Author '{}' listed in Copyright but not in git history".format(author))
+            return
+
+        if email not in git_history[author]['emails']:
+            yield CheckResult("Email '{} <{}>' listed in Copyright not in git history".format(author, values['email']))
+
+        git_year_range = min(git_history[author]['years']), max(git_history[author]['years'])
+        cop_year_range = values['year_range']
+        if git_year_range != cop_year_range:
+            yield CheckResult("Author '{}' listed in Copyright with years '{} - {}' but git history says '{} - {}'".format(author, cop_year_range[0], cop_year_range[1], git_year_range[0], git_year_range[1]))
+
+    yield CheckResult(filepath)
+
+
+if len(sys.argv) != 2:
+    print("Git repository path not specified")
+    sys.exit(2)
+
+REPO_PATH = os.path.abspath(sys.argv[1])
+
+if not os.path.exists(os.path.join(REPO_PATH, ".git")):
+    print(".git not found in {}".format(REPO_PATH))
+    sys.exit(2)
+
+if not shutil.which("git"):
+    print("git executable not found in PATH")
+    sys.exit(2)
+
+check = Check()
+check.register(no_CR)
+check.register(copyright)
+check.register(clang_format)
+
+res = check.run(REPO_PATH)
+sys.exit(0 if res else 1)