From 9ff44136ab491ebb196b6fc0f4e14adc0cf7a5e1 Mon Sep 17 00:00:00 2001 From: nyanloutre Date: Sat, 10 Nov 2018 15:23:46 +0100 Subject: [PATCH] =?UTF-8?q?nouveau=20syst=C3=A8me=20de=20CI=20utilisant=20?= =?UTF-8?q?les=20webhooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- overlays/site-max.nix | 5 -- services/python-ci.nix | 41 +++++++++ services/python-ci.py | 153 ++++++++++++++++++++++++++++++++++ services/site-max.nix | 42 +--------- services/site-musique.nix | 37 +------- systems/LoutreOS/services.nix | 4 + 6 files changed, 200 insertions(+), 82 deletions(-) delete mode 100644 overlays/site-max.nix create mode 100644 services/python-ci.nix create mode 100755 services/python-ci.py diff --git a/overlays/site-max.nix b/overlays/site-max.nix deleted file mode 100644 index b52cea9e1..000000000 --- a/overlays/site-max.nix +++ /dev/null @@ -1,5 +0,0 @@ -self: super: - -{ - site-max = super.callPackage ../pkgs/site-max { }; -} diff --git a/services/python-ci.nix b/services/python-ci.nix new file mode 100644 index 000000000..2523d8f65 --- /dev/null +++ b/services/python-ci.nix @@ -0,0 +1,41 @@ +{lib, config, pkgs, ... }: + +with lib; + +let + cfg = config.services.python-ci; +in +{ + options.services.python-ci = { + enable = mkEnableOption "Service de CI Nix écrit en Python"; + }; + + config = mkIf cfg.enable { + + systemd.services.python-ci = { + description = "CI Nix en Python"; + requires = ["network-online.target"]; + wantedBy = ["multi-user.target"]; + environment = { HOME = "/var/lib/python-ci"; NIX_PATH = concatStringsSep ":" config.nix.nixPath; NIXPKGS_ALLOW_UNFREE = "1";}; + path = with pkgs;[ nix gnutar gzip ]; + serviceConfig = { + DynamicUser = true; + StateDirectory = "python-ci"; + RuntimeDirectory = "python-ci"; + RuntimeDirectoryPreserve = "yes"; + ExecStart = with pkgs; + let env = python3Packages.python.buildEnv.override { + extraLibs = with python3Packages;[ pyramid python-gitlab ]; + ignoreCollisions = true; + }; + in "${pkgs.writeShellScriptBin "run.sh" '' + ${env}/bin/python ${pkgs.writeScript "python-ci.py" "${readFile ./python-ci.py}"} --port 52350 \ + --secret /var/lib/python-ci/secret --gitlab-token /var/lib/python-ci/gitlab_token \ + --gitea-token /var/lib/python-ci/gitea_token --output /run/python-ci + ''}/bin/run.sh"; + }; + }; + + }; + +} diff --git a/services/python-ci.py b/services/python-ci.py new file mode 100755 index 000000000..825f40226 --- /dev/null +++ b/services/python-ci.py @@ -0,0 +1,153 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p "python3.withPackages(ps: [ps.pyramid ps.python-gitlab])" +from wsgiref.simple_server import make_server +from pyramid.config import Configurator +from pyramid.view import view_config, view_defaults +from pyramid.httpexceptions import HTTPNotFound +from subprocess import check_call, CalledProcessError +import urllib.request +import tarfile +from tempfile import TemporaryDirectory +from multiprocessing import Pool +from gitlab import Gitlab +import urllib.request +import json +import argparse + + +def gitlab_build(payload, gl): + commit = gl.projects.get(payload['project']['path_with_namespace']).commits.get(payload['checkout_sha']) + + commit.statuses.create({'state': 'running', 'name': 'Python CI'}) + print("push from " + payload['user_name']) + print("repo: " + payload['project']['path_with_namespace']) + print("commit: " + payload['checkout_sha']) + temp_dir = TemporaryDirectory() + repo_dir = temp_dir.name + '/' + payload['project']['name'] + '-' + payload['checkout_sha'] + archive_url = payload['project']['web_url'] + '/-/archive/' + payload['checkout_sha'] + \ + '/' + payload['project']['name'] + '-' + payload['checkout_sha'] + '.tar.gz' + + with urllib.request.urlopen(archive_url) as gitlab_archive: + with tarfile.open(fileobj=gitlab_archive, mode='r|gz') as gitlab_repo_files: + gitlab_repo_files.extractall(path=temp_dir.name) + + check_call(['ls', '-lha', repo_dir]) + + try: + check_call(['nix-build', '-o', args.output + '/' + payload['project']['path_with_namespace'], repo_dir]) + except CalledProcessError: + commit.statuses.create({'state': 'failed', 'name': 'Python CI'}) + print("erreur build") + else: + commit.statuses.create({'state': 'success', 'name': 'Python CI'}) + print("build terminé") + + +@view_defaults( + route_name="gitlab_payload", renderer="json", request_method="POST" +) +class GitlabHook(object): + + def __init__(self, request): + self.request = request + self.payload = self.request.json + self.whitelist = ['nyanloutre/site-musique'] + self.secret = open(args.secret, 'r').readline().splitlines()[0] + self.gitlab_token = open(args.gitlab_token, 'r').readline().splitlines()[0] + self.gl = Gitlab('https://gitlab.com', private_token=self.gitlab_token) + + @view_config(header="X-Gitlab-Event:Push Hook") + def push_hook(self): + if self.payload['project']['path_with_namespace'] in self.whitelist and self.request.headers['X-Gitlab-Token'] == self.secret: + self.gl.projects.get(self.payload['project']['path_with_namespace']).commits.get(self.payload['checkout_sha']).statuses.create({'state': 'pending', 'name': 'Python CI'}) + pool.apply_async(gitlab_build, (self.payload, self.gl)) + return "build started" + else: + raise HTTPNotFound + + +def gitea_status_update(repo, commit, token, status): + url = 'https://gitea.nyanlout.re/api/v1/repos/' + repo + '/statuses/' + commit + print(url) + req = urllib.request.Request(url) + req.add_header('Content-Type', 'application/json; charset=utf-8') + req.add_header('accept', 'application/json') + req.add_header('Authorization', 'token ' + token) + + jsondata = json.dumps({'state': status}).encode('utf-8') + req.add_header('Content-Length', len(jsondata)) + + urllib.request.urlopen(req, jsondata) + +def gitea_build(payload, token): + commit = payload['after'] + repo = payload['repository']['full_name'] + + gitea_status_update(repo, commit, token, 'pending') + + print("push from " + payload['pusher']['username']) + print("repo: " + repo) + print("commit: " + commit) + temp_dir = TemporaryDirectory() + repo_dir = temp_dir.name + '/' + payload['repository']['name'] + archive_url = payload['repository']['html_url'] + '/archive/' + commit + '.tar.gz' + + with urllib.request.urlopen(archive_url) as gitea_archive: + with tarfile.open(fileobj=gitea_archive, mode='r|gz') as gitea_repo_files: + gitea_repo_files.extractall(path=temp_dir.name) + + check_call(['ls', '-lha', repo_dir]) + + try: + check_call(['nix-build', '-o', args.output + '/' + repo, repo_dir]) + except CalledProcessError: + gitea_status_update(repo, commit, token, 'failure') + print("erreur build") + else: + gitea_status_update(repo, commit, token, 'success') + print("build terminé") + + +@view_defaults( + route_name="gitea_payload", renderer="json", request_method="POST" +) +class GiteaHook(object): + + def __init__(self, request): + self.request = request + self.payload = self.request.json + self.whitelist = ['nyanloutre/site-musique', 'nyanloutre/site-max'] + self.secret = open(args.secret, 'r').readline().splitlines()[0] + self.gitea_token = open(args.gitea_token, 'r').readline().splitlines()[0] + + @view_config(header="X-Gitea-Event:push") + def push_hook(self): + if self.payload['repository']['full_name'] in self.whitelist and self.payload['secret'] == self.secret: + pool.apply_async(gitea_build, (self.payload, self.gitea_token)) + return "build started" + else: + raise HTTPNotFound + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='CI server') + parser.add_argument('--address', help='listening address', default='127.0.0.1') + parser.add_argument('--port', type=int, help='listening port') + parser.add_argument('--output', help='output directory') + parser.add_argument('--secret', help='repo secret file') + parser.add_argument('--gitlab-token', help='gitlab token file') + parser.add_argument('--gitea-token', help='gitea token file') + args = parser.parse_args() + + pool = Pool(1) + + config = Configurator() + + config.add_route("gitlab_payload", "/gitlab_payload") + config.add_route("gitea_payload", "/gitea_payload") + config.scan() + + app = config.make_wsgi_app() + server = make_server(args.address, args.port, app) + print('listening ...') + server.serve_forever() diff --git a/services/site-max.nix b/services/site-max.nix index f5cccbe49..da900b12e 100644 --- a/services/site-max.nix +++ b/services/site-max.nix @@ -24,10 +24,6 @@ in config = mkIf cfg.enable { - nixpkgs.overlays = [ - (import ../overlays/site-max.nix) - ]; - services.haproxy-acme.services = { ${cfg.domaine} = { ip = "127.0.0.1"; port = cfg.port; auth = false; }; }; @@ -37,46 +33,10 @@ in "max" = { listen = [ { addr = "127.0.0.1"; port = cfg.port; } ]; locations."/" = { - root = "/run/site-max/result"; + root = "/run/python-ci/nyanloutre/site-max"; }; }; }; }; - - systemd.services.build-site-max = { - description = "Compilation du site de Max Spiegel"; - requires = ["network-online.target"]; - path = with pkgs;[ git nix ]; - environment = { HOME = "/var/lib/site-max"; NIX_PATH = "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs"; }; - - serviceConfig = { - DynamicUser = true; - RuntimeDirectory = "site-max"; - RuntimeDirectoryPreserve = "yes"; - CacheDirectory = "site-max"; - Type = "oneshot"; - ExecStart = "${pkgs.writeShellScriptBin "build.sh" '' - set -x - set -e - GIT_CLONE_DIR=/var/cache/site-max - - if [ ! -d $GIT_CLONE_DIR/.git ]; then - git clone --depth 1 https://github.com/nyanloutre/site-max.git $GIT_CLONE_DIR - else - git -C $GIT_CLONE_DIR pull - fi - - NIXPKGS_ALLOW_UNFREE=1 nix-build -o /run/site-max/result $GIT_CLONE_DIR - ''}/bin/build.sh"; - }; - }; - - systemd.timers.build-site-max = { - description = "Timer de compilation du site de Max"; - requires = ["network-online.target"]; - wantedBy = ["multi-user.target"]; - timerConfig = { OnCalendar = "*:0/5"; Unit = "build-site-max.service"; }; - }; - }; } diff --git a/services/site-musique.nix b/services/site-musique.nix index ef1fbd29f..62cee74a0 100644 --- a/services/site-musique.nix +++ b/services/site-musique.nix @@ -32,7 +32,7 @@ in "musique" = { listen = [ { addr = "127.0.0.1"; port = cfg.port; } ]; locations."/" = { - root = "/run/site-musique/result"; + root = "/run/python-ci/nyanloutre/site-musique"; index = "index.php"; extraConfig = '' location ~* \.php$ { @@ -62,40 +62,5 @@ in php_admin_flag[log_errors] = on catch_workers_output = yes ''; - - systemd.services.build-site-musique = { - description = "Compilation du site de la musique"; - requires = ["network-online.target"]; - path = with pkgs;[ git nix ]; - environment = { HOME = "/var/lib/site-musique"; NIX_PATH = "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs"; }; - - serviceConfig = { - DynamicUser = true; - RuntimeDirectory = "site-musique"; - RuntimeDirectoryPreserve = "yes"; - CacheDirectory = "site-musique"; - Type = "oneshot"; - ExecStart = "${pkgs.writeShellScriptBin "build.sh" '' - set -x - set -e - GIT_CLONE_DIR=/var/cache/site-musique - - if [ ! -d $GIT_CLONE_DIR/.git ]; then - git clone --depth 1 https://gitlab.com/nyanloutre/site-musique.git $GIT_CLONE_DIR - else - git -C $GIT_CLONE_DIR pull - fi - - NIXPKGS_ALLOW_UNFREE=1 nix-build -o /run/site-musique/result $GIT_CLONE_DIR - ''}/bin/build.sh"; - }; - }; - - systemd.timers.build-site-musique = { - description = "Timer de compilation du site de la musique"; - requires = ["network-online.target"]; - wantedBy = ["multi-user.target"]; - timerConfig = { OnCalendar = "*:0/5"; Unit = "build-site-musique.service"; }; - }; }; } diff --git a/systems/LoutreOS/services.nix b/systems/LoutreOS/services.nix index c785a5db1..3a26b772c 100644 --- a/systems/LoutreOS/services.nix +++ b/systems/LoutreOS/services.nix @@ -18,6 +18,7 @@ in ../../services/site-musique.nix ../../services/site-max.nix ../../services/auto-pr.nix + ../../services/python-ci.nix ../../containers/vsftpd.nix ]; @@ -56,6 +57,7 @@ in "matrix.${domaine}" = { ip = "127.0.0.1"; port = 8008; auth = false; }; "pgmanage.${domaine}" = { ip = "127.0.0.1"; port = pgmanage_port; auth = true; }; "gitea.${domaine}" = { ip = "127.0.0.1"; port = 3001; auth = false; }; + "ci.${domaine}" = { ip = "127.0.0.1"; port = 52350; auth = false; }; }; }; @@ -356,6 +358,8 @@ in }; auto-pr.enable = true; + + python-ci.enable = true; }; /*