nixos-config/services/python-ci.py

#! /usr/bin/env nix-shell
#! nix-shell -i python3 -p "python3.withPackages(ps: [ps.pyramid ps.python-gitlab])"
from wsgiref.simple_server import make_server
from pyramid.config import Configurator
from pyramid.view import view_config, view_defaults
from pyramid.httpexceptions import HTTPNotFound
from subprocess import check_call, CalledProcessError
import urllib.request
import tarfile
from tempfile import TemporaryDirectory
from multiprocessing import Pool
from gitlab import Gitlab
import urllib.request
import json
import argparse
import hmac
import hashlib


def gitlab_build(payload, gl):
    commit = gl.projects.get(payload['project']['path_with_namespace']).commits.get(payload['checkout_sha'])

    commit.statuses.create({'state': 'running', 'name': 'Python CI'})
    print("push from " + payload['user_name'])
    print("repo: " + payload['project']['path_with_namespace'])
    print("commit: " + payload['checkout_sha'])
    temp_dir = TemporaryDirectory()
    repo_dir = temp_dir.name + '/' + payload['project']['name'] + '-' + payload['checkout_sha']
    archive_url = payload['project']['web_url'] + '/-/archive/' + payload['checkout_sha'] + \
                  '/' + payload['project']['name'] + '-' + payload['checkout_sha'] + '.tar.gz'

    with urllib.request.urlopen(archive_url) as gitlab_archive:
        with tarfile.open(fileobj=gitlab_archive, mode='r|gz') as gitlab_repo_files:
            gitlab_repo_files.extractall(path=temp_dir.name)

    check_call(['ls', '-lha', repo_dir])

    try:
        check_call(['nix-build', '-o', args.output + '/' + payload['project']['path_with_namespace'], repo_dir])
    except CalledProcessError:
        commit.statuses.create({'state': 'failed', 'name': 'Python CI'})
        print("erreur build")
    else:
        commit.statuses.create({'state': 'success', 'name': 'Python CI'})
        print("build terminé")


@view_defaults(
        route_name="gitlab_payload", renderer="json", request_method="POST"
)
class GitlabHook(object):

    def __init__(self, request):
        self.request = request
        self.payload = self.request.json
        self.whitelist = ['nyanloutre/site-musique']
        self.secret = open(args.secret, 'r').readline().splitlines()[0]
        self.gitlab_token = open(args.gitlab_token, 'r').readline().splitlines()[0]
        self.gl = Gitlab('https://gitlab.com', private_token=self.gitlab_token)

    @view_config(header="X-Gitlab-Event:Push Hook")
    def push_hook(self):
        if self.payload['project']['path_with_namespace'] in self.whitelist and self.request.headers['X-Gitlab-Token'] == self.secret:
            self.gl.projects.get(self.payload['project']['path_with_namespace']).commits.get(self.payload['checkout_sha']).statuses.create({'state': 'pending', 'name': 'Python CI'})
            pool.apply_async(gitlab_build, (self.payload, self.gl))
            return "build started"
        else:
            raise HTTPNotFound


def gitea_status_update(repo, commit, token, status):
    url = 'https://gitea.nyanlout.re/api/v1/repos/' + repo + '/statuses/' + commit
    print(url)
    req = urllib.request.Request(url)
    req.add_header('Content-Type', 'application/json; charset=utf-8')
    req.add_header('accept', 'application/json')
    req.add_header('Authorization', 'token ' + token)

    jsondata = json.dumps({'state': status}).encode('utf-8')
    req.add_header('Content-Length', len(jsondata))

    urllib.request.urlopen(req, jsondata)

def gitea_build(payload, token):
    commit = payload['after']
    repo = payload['repository']['full_name']

    gitea_status_update(repo, commit, token, 'pending')

    print("push from " + payload['pusher']['username'])
    print("repo: " + repo)
    print("commit: " + commit)
    temp_dir = TemporaryDirectory()
    repo_dir = temp_dir.name + '/' + payload['repository']['name']
    archive_url = payload['repository']['html_url'] + '/archive/' + commit + '.tar.gz'

    with urllib.request.urlopen(archive_url) as gitea_archive:
        with tarfile.open(fileobj=gitea_archive, mode='r|gz') as gitea_repo_files:
            gitea_repo_files.extractall(path=temp_dir.name)

    check_call(['ls', '-lha', repo_dir])

    try:
        check_call(['nix-build', '-o', args.output + '/' + repo, repo_dir])
    except CalledProcessError:
        gitea_status_update(repo, commit, token, 'failure')
        print("erreur build")
    else:
        gitea_status_update(repo, commit, token, 'success')
        print("build terminé")


@view_defaults(
        route_name="gitea_payload", renderer="json", request_method="POST"
)
class GiteaHook(object):
    def __init__(self, request):
        self.payload = request.json
        self.whitelist = ['nyanloutre/site-musique', 'nyanloutre/site-max']
        self.gitea_token = open(args.gitea_token, 'r').readline().strip()

    @view_config(header=["X-Gitea-Event:push", "X-Gitea-Signature"], check_hmac=True)
    def push_hook(self):
        if self.payload['repository']['full_name'] in self.whitelist:
            pool.apply_async(gitea_build, (self.payload, self.gitea_token))
            return "build started"
        else:
            raise HTTPNotFound


class CheckHmacPredicate(object):
    def __init__(self, val, info):
        self.secret = open(args.secret, 'r').readline().strip().encode()

    def text(self):
        return 'HMAC checking enabled'

    phash = text

    def __call__(self, context, request):
        payload_signature = hmac.new(self.secret, request.body, hashlib.sha256).hexdigest()
        return hmac.compare_digest(request.headers["X-Gitea-Signature"], payload_signature)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='CI server')
    parser.add_argument('--address', help='listening address', default='127.0.0.1')
    parser.add_argument('--port', type=int, help='listening port')
    parser.add_argument('--output', help='output directory')
    parser.add_argument('--secret', help='repo secret file')
    parser.add_argument('--gitlab-token', help='gitlab token file')
    parser.add_argument('--gitea-token', help='gitea token file')
    args = parser.parse_args()


    pool = Pool(1)

    config = Configurator()

    config.add_view_predicate('check_hmac', CheckHmacPredicate)

    config.add_route("gitlab_payload", "/gitlab_payload")
    config.add_route("gitea_payload", "/gitea_payload")
    config.scan()

    app = config.make_wsgi_app()
    server = make_server(args.address, args.port, app)
    print('listening ...')
    server.serve_forever()