forked from nyanloutre/nixos-config
nouveau système de CI utilisant les webhooks
This commit is contained in:
parent
aa57b7b325
commit
9ff44136ab
|
|
@ -1,5 +0,0 @@
|
|||
self: super:
|
||||
|
||||
{
|
||||
site-max = super.callPackage ../pkgs/site-max { };
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{lib, config, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.services.python-ci;
|
||||
in
|
||||
{
|
||||
options.services.python-ci = {
|
||||
enable = mkEnableOption "Service de CI Nix écrit en Python";
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
|
||||
systemd.services.python-ci = {
|
||||
description = "CI Nix en Python";
|
||||
requires = ["network-online.target"];
|
||||
wantedBy = ["multi-user.target"];
|
||||
environment = { HOME = "/var/lib/python-ci"; NIX_PATH = concatStringsSep ":" config.nix.nixPath; NIXPKGS_ALLOW_UNFREE = "1";};
|
||||
path = with pkgs;[ nix gnutar gzip ];
|
||||
serviceConfig = {
|
||||
DynamicUser = true;
|
||||
StateDirectory = "python-ci";
|
||||
RuntimeDirectory = "python-ci";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
ExecStart = with pkgs;
|
||||
let env = python3Packages.python.buildEnv.override {
|
||||
extraLibs = with python3Packages;[ pyramid python-gitlab ];
|
||||
ignoreCollisions = true;
|
||||
};
|
||||
in "${pkgs.writeShellScriptBin "run.sh" ''
|
||||
${env}/bin/python ${pkgs.writeScript "python-ci.py" "${readFile ./python-ci.py}"} --port 52350 \
|
||||
--secret /var/lib/python-ci/secret --gitlab-token /var/lib/python-ci/gitlab_token \
|
||||
--gitea-token /var/lib/python-ci/gitea_token --output /run/python-ci
|
||||
''}/bin/run.sh";
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i python3 -p "python3.withPackages(ps: [ps.pyramid ps.python-gitlab])"
|
||||
from wsgiref.simple_server import make_server
|
||||
from pyramid.config import Configurator
|
||||
from pyramid.view import view_config, view_defaults
|
||||
from pyramid.httpexceptions import HTTPNotFound
|
||||
from subprocess import check_call, CalledProcessError
|
||||
import urllib.request
|
||||
import tarfile
|
||||
from tempfile import TemporaryDirectory
|
||||
from multiprocessing import Pool
|
||||
from gitlab import Gitlab
|
||||
import urllib.request
|
||||
import json
|
||||
import argparse
|
||||
|
||||
|
||||
def gitlab_build(payload, gl):
|
||||
commit = gl.projects.get(payload['project']['path_with_namespace']).commits.get(payload['checkout_sha'])
|
||||
|
||||
commit.statuses.create({'state': 'running', 'name': 'Python CI'})
|
||||
print("push from " + payload['user_name'])
|
||||
print("repo: " + payload['project']['path_with_namespace'])
|
||||
print("commit: " + payload['checkout_sha'])
|
||||
temp_dir = TemporaryDirectory()
|
||||
repo_dir = temp_dir.name + '/' + payload['project']['name'] + '-' + payload['checkout_sha']
|
||||
archive_url = payload['project']['web_url'] + '/-/archive/' + payload['checkout_sha'] + \
|
||||
'/' + payload['project']['name'] + '-' + payload['checkout_sha'] + '.tar.gz'
|
||||
|
||||
with urllib.request.urlopen(archive_url) as gitlab_archive:
|
||||
with tarfile.open(fileobj=gitlab_archive, mode='r|gz') as gitlab_repo_files:
|
||||
gitlab_repo_files.extractall(path=temp_dir.name)
|
||||
|
||||
check_call(['ls', '-lha', repo_dir])
|
||||
|
||||
try:
|
||||
check_call(['nix-build', '-o', args.output + '/' + payload['project']['path_with_namespace'], repo_dir])
|
||||
except CalledProcessError:
|
||||
commit.statuses.create({'state': 'failed', 'name': 'Python CI'})
|
||||
print("erreur build")
|
||||
else:
|
||||
commit.statuses.create({'state': 'success', 'name': 'Python CI'})
|
||||
print("build terminé")
|
||||
|
||||
|
||||
@view_defaults(
|
||||
route_name="gitlab_payload", renderer="json", request_method="POST"
|
||||
)
|
||||
class GitlabHook(object):
|
||||
|
||||
def __init__(self, request):
|
||||
self.request = request
|
||||
self.payload = self.request.json
|
||||
self.whitelist = ['nyanloutre/site-musique']
|
||||
self.secret = open(args.secret, 'r').readline().splitlines()[0]
|
||||
self.gitlab_token = open(args.gitlab_token, 'r').readline().splitlines()[0]
|
||||
self.gl = Gitlab('https://gitlab.com', private_token=self.gitlab_token)
|
||||
|
||||
@view_config(header="X-Gitlab-Event:Push Hook")
|
||||
def push_hook(self):
|
||||
if self.payload['project']['path_with_namespace'] in self.whitelist and self.request.headers['X-Gitlab-Token'] == self.secret:
|
||||
self.gl.projects.get(self.payload['project']['path_with_namespace']).commits.get(self.payload['checkout_sha']).statuses.create({'state': 'pending', 'name': 'Python CI'})
|
||||
pool.apply_async(gitlab_build, (self.payload, self.gl))
|
||||
return "build started"
|
||||
else:
|
||||
raise HTTPNotFound
|
||||
|
||||
|
||||
def gitea_status_update(repo, commit, token, status):
|
||||
url = 'https://gitea.nyanlout.re/api/v1/repos/' + repo + '/statuses/' + commit
|
||||
print(url)
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('Content-Type', 'application/json; charset=utf-8')
|
||||
req.add_header('accept', 'application/json')
|
||||
req.add_header('Authorization', 'token ' + token)
|
||||
|
||||
jsondata = json.dumps({'state': status}).encode('utf-8')
|
||||
req.add_header('Content-Length', len(jsondata))
|
||||
|
||||
urllib.request.urlopen(req, jsondata)
|
||||
|
||||
def gitea_build(payload, token):
|
||||
commit = payload['after']
|
||||
repo = payload['repository']['full_name']
|
||||
|
||||
gitea_status_update(repo, commit, token, 'pending')
|
||||
|
||||
print("push from " + payload['pusher']['username'])
|
||||
print("repo: " + repo)
|
||||
print("commit: " + commit)
|
||||
temp_dir = TemporaryDirectory()
|
||||
repo_dir = temp_dir.name + '/' + payload['repository']['name']
|
||||
archive_url = payload['repository']['html_url'] + '/archive/' + commit + '.tar.gz'
|
||||
|
||||
with urllib.request.urlopen(archive_url) as gitea_archive:
|
||||
with tarfile.open(fileobj=gitea_archive, mode='r|gz') as gitea_repo_files:
|
||||
gitea_repo_files.extractall(path=temp_dir.name)
|
||||
|
||||
check_call(['ls', '-lha', repo_dir])
|
||||
|
||||
try:
|
||||
check_call(['nix-build', '-o', args.output + '/' + repo, repo_dir])
|
||||
except CalledProcessError:
|
||||
gitea_status_update(repo, commit, token, 'failure')
|
||||
print("erreur build")
|
||||
else:
|
||||
gitea_status_update(repo, commit, token, 'success')
|
||||
print("build terminé")
|
||||
|
||||
|
||||
@view_defaults(
|
||||
route_name="gitea_payload", renderer="json", request_method="POST"
|
||||
)
|
||||
class GiteaHook(object):
|
||||
|
||||
def __init__(self, request):
|
||||
self.request = request
|
||||
self.payload = self.request.json
|
||||
self.whitelist = ['nyanloutre/site-musique', 'nyanloutre/site-max']
|
||||
self.secret = open(args.secret, 'r').readline().splitlines()[0]
|
||||
self.gitea_token = open(args.gitea_token, 'r').readline().splitlines()[0]
|
||||
|
||||
@view_config(header="X-Gitea-Event:push")
|
||||
def push_hook(self):
|
||||
if self.payload['repository']['full_name'] in self.whitelist and self.payload['secret'] == self.secret:
|
||||
pool.apply_async(gitea_build, (self.payload, self.gitea_token))
|
||||
return "build started"
|
||||
else:
|
||||
raise HTTPNotFound
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='CI server')
|
||||
parser.add_argument('--address', help='listening address', default='127.0.0.1')
|
||||
parser.add_argument('--port', type=int, help='listening port')
|
||||
parser.add_argument('--output', help='output directory')
|
||||
parser.add_argument('--secret', help='repo secret file')
|
||||
parser.add_argument('--gitlab-token', help='gitlab token file')
|
||||
parser.add_argument('--gitea-token', help='gitea token file')
|
||||
args = parser.parse_args()
|
||||
|
||||
pool = Pool(1)
|
||||
|
||||
config = Configurator()
|
||||
|
||||
config.add_route("gitlab_payload", "/gitlab_payload")
|
||||
config.add_route("gitea_payload", "/gitea_payload")
|
||||
config.scan()
|
||||
|
||||
app = config.make_wsgi_app()
|
||||
server = make_server(args.address, args.port, app)
|
||||
print('listening ...')
|
||||
server.serve_forever()
|
||||
|
|
@ -24,10 +24,6 @@ in
|
|||
|
||||
config = mkIf cfg.enable {
|
||||
|
||||
nixpkgs.overlays = [
|
||||
(import ../overlays/site-max.nix)
|
||||
];
|
||||
|
||||
services.haproxy-acme.services = {
|
||||
${cfg.domaine} = { ip = "127.0.0.1"; port = cfg.port; auth = false; };
|
||||
};
|
||||
|
|
@ -37,46 +33,10 @@ in
|
|||
"max" = {
|
||||
listen = [ { addr = "127.0.0.1"; port = cfg.port; } ];
|
||||
locations."/" = {
|
||||
root = "/run/site-max/result";
|
||||
root = "/run/python-ci/nyanloutre/site-max";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.build-site-max = {
|
||||
description = "Compilation du site de Max Spiegel";
|
||||
requires = ["network-online.target"];
|
||||
path = with pkgs;[ git nix ];
|
||||
environment = { HOME = "/var/lib/site-max"; NIX_PATH = "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs"; };
|
||||
|
||||
serviceConfig = {
|
||||
DynamicUser = true;
|
||||
RuntimeDirectory = "site-max";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
CacheDirectory = "site-max";
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.writeShellScriptBin "build.sh" ''
|
||||
set -x
|
||||
set -e
|
||||
GIT_CLONE_DIR=/var/cache/site-max
|
||||
|
||||
if [ ! -d $GIT_CLONE_DIR/.git ]; then
|
||||
git clone --depth 1 https://github.com/nyanloutre/site-max.git $GIT_CLONE_DIR
|
||||
else
|
||||
git -C $GIT_CLONE_DIR pull
|
||||
fi
|
||||
|
||||
NIXPKGS_ALLOW_UNFREE=1 nix-build -o /run/site-max/result $GIT_CLONE_DIR
|
||||
''}/bin/build.sh";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.build-site-max = {
|
||||
description = "Timer de compilation du site de Max";
|
||||
requires = ["network-online.target"];
|
||||
wantedBy = ["multi-user.target"];
|
||||
timerConfig = { OnCalendar = "*:0/5"; Unit = "build-site-max.service"; };
|
||||
};
|
||||
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ in
|
|||
"musique" = {
|
||||
listen = [ { addr = "127.0.0.1"; port = cfg.port; } ];
|
||||
locations."/" = {
|
||||
root = "/run/site-musique/result";
|
||||
root = "/run/python-ci/nyanloutre/site-musique";
|
||||
index = "index.php";
|
||||
extraConfig = ''
|
||||
location ~* \.php$ {
|
||||
|
|
@ -62,40 +62,5 @@ in
|
|||
php_admin_flag[log_errors] = on
|
||||
catch_workers_output = yes
|
||||
'';
|
||||
|
||||
systemd.services.build-site-musique = {
|
||||
description = "Compilation du site de la musique";
|
||||
requires = ["network-online.target"];
|
||||
path = with pkgs;[ git nix ];
|
||||
environment = { HOME = "/var/lib/site-musique"; NIX_PATH = "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs"; };
|
||||
|
||||
serviceConfig = {
|
||||
DynamicUser = true;
|
||||
RuntimeDirectory = "site-musique";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
CacheDirectory = "site-musique";
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.writeShellScriptBin "build.sh" ''
|
||||
set -x
|
||||
set -e
|
||||
GIT_CLONE_DIR=/var/cache/site-musique
|
||||
|
||||
if [ ! -d $GIT_CLONE_DIR/.git ]; then
|
||||
git clone --depth 1 https://gitlab.com/nyanloutre/site-musique.git $GIT_CLONE_DIR
|
||||
else
|
||||
git -C $GIT_CLONE_DIR pull
|
||||
fi
|
||||
|
||||
NIXPKGS_ALLOW_UNFREE=1 nix-build -o /run/site-musique/result $GIT_CLONE_DIR
|
||||
''}/bin/build.sh";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.build-site-musique = {
|
||||
description = "Timer de compilation du site de la musique";
|
||||
requires = ["network-online.target"];
|
||||
wantedBy = ["multi-user.target"];
|
||||
timerConfig = { OnCalendar = "*:0/5"; Unit = "build-site-musique.service"; };
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ in
|
|||
../../services/site-musique.nix
|
||||
../../services/site-max.nix
|
||||
../../services/auto-pr.nix
|
||||
../../services/python-ci.nix
|
||||
../../containers/vsftpd.nix
|
||||
];
|
||||
|
||||
|
|
@ -56,6 +57,7 @@ in
|
|||
"matrix.${domaine}" = { ip = "127.0.0.1"; port = 8008; auth = false; };
|
||||
"pgmanage.${domaine}" = { ip = "127.0.0.1"; port = pgmanage_port; auth = true; };
|
||||
"gitea.${domaine}" = { ip = "127.0.0.1"; port = 3001; auth = false; };
|
||||
"ci.${domaine}" = { ip = "127.0.0.1"; port = 52350; auth = false; };
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -356,6 +358,8 @@ in
|
|||
};
|
||||
|
||||
auto-pr.enable = true;
|
||||
|
||||
python-ci.enable = true;
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
|
|||
Loading…
Reference in New Issue