Merge pull request #102 from Liqwid-Labs/connor/bench-check

ci: check benchmark report is up-to-date

.github/workflows/integrate.yaml

@@ -37,6 +37,28 @@ jobs:
       - run: nix build .#checks.x86_64-linux.formatCheck
         name: Run 'formatCheck' from flake.nix
 
+  check-bench:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.4.0
+
+      - uses: cachix/install-nix-action@v16
+        name: Set up Nix and IOHK caches
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+          extra_nix_config: |
+            trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+            substituters = https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/
+            experimental-features = nix-command flakes
+
+      - uses: cachix/cachix-action@v10
+        with:
+          name: mlabs
+          authToken: ${{ secrets.CACHIX_KEY }}
+
+      - run: nix build .#checks.x86_64-linux.benchCheck
+        name: Run 'benchCheck' from flake.nix
+
   check-build:
     runs-on: ubuntu-latest
     steps:
@@ -69,35 +91,33 @@ jobs:
       - name: Build the project
         run: nix build .#check.x86_64-linux
 
-
-
   haddock:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2.4.0
+      - uses: actions/checkout@v2.4.0
 
-    - uses: cachix/install-nix-action@v16
+      - uses: cachix/install-nix-action@v16
-      name: Set up Nix and IOHK caches
+        name: Set up Nix and IOHK caches
-      with:
+        with:
-        nix_path: nixpkgs=channel:nixos-unstable
+          nix_path: nixpkgs=channel:nixos-unstable
-        extra_nix_config: |
+          extra_nix_config: |
-          trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+            trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-          substituters = https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/
+            substituters = https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/
-          experimental-features = nix-command flakes
+            experimental-features = nix-command flakes
 
-    - uses: cachix/cachix-action@v10
+      - uses: cachix/cachix-action@v10
-      with:
+        with:
-        name: mlabs
+          name: mlabs
-        authToken: ${{ secrets.CACHIX_KEY }}
+          authToken: ${{ secrets.CACHIX_KEY }}
 
-    - run: nix build .#packages.x86_64-linux.haddock
+      - run: nix build .#packages.x86_64-linux.haddock
-      name: Run 'haddock' from flake.nix
+        name: Run 'haddock' from flake.nix
 
-    # This publishes the haddock result to the branch 'gh-pages',
+      # This publishes the haddock result to the branch 'gh-pages',
-    # which is set to automatically deploy to https://liqwid-labs.github.io/agora/.
+      # which is set to automatically deploy to https://liqwid-labs.github.io/agora/.
-    - name: Publish Documentation
+      - name: Publish Documentation
-      uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v3
-      if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/master'
-      with:
+        with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: ./result/agora/html
+          publish_dir: ./result/agora/html

Makefile

@@ -1,7 +1,7 @@
 # This really ought to be `/usr/bin/env bash`, but nix flakes don't like that.
 SHELL := /bin/sh
 
-.PHONY: hoogle format haddock usage tag format_nix format_haskell format_check lint ps_bridge
+.PHONY: hoogle format haddock usage tag format_nix format_haskell format_check lint ps_bridge bench bench_check
 
 usage:
 	@echo "usage: make <command> [OPTIONS]"
@@ -16,6 +16,8 @@ usage:
 	@echo "  format_check -- Check if all haskell stuff have been formatted correctly"
 	@echo "  lint -- Get hlint suggestions for project"
 	@echo "  ps_bridge -- Generate purescript bridge files"
+	@echo "  bench -- Generate bench report bench.csv"
+	@echo "  bench_check -- Check if bench report is up-to-date"
 
 hoogle:
 	pkill hoogle || true
@@ -51,3 +53,16 @@ lint:
 PS_BRIDGE_OUTPUT_DIR := agora-purescript-bridge/
 ps_bridge:
 	cabal run exe:agora-purescript-bridge -- -o $(PS_BRIDGE_OUTPUT_DIR)
+
+bench:
+	cabal run agora-bench
+
+BENCH_TMPDIR := $(shell mktemp -d)
+BENCH_TMPFILE := $(BENCH_TMPDIR)/bench.csv
+bench_check:
+	(cabal run agora-bench -- -o "$(BENCH_TMPFILE)" \
+		|| $(bench) -o "$(BENCH_TMPFILE)") >> /dev/null
+	diff bench.csv $(BENCH_TMPFILE) \
+		|| (echo "bench.csv is outdated"; exit 1)
+	# TODO: do the clean-up even if `diff` fails.
+	rm -rf $(BENCH_TMPDIR)

agora-bench/Main.hs

@@ -4,6 +4,7 @@ import Bench (specificationTreeToBenchmarks)
 import Data.Csv (encodeDefaultOrderedByName)
 import Data.Text.Lazy.Encoding (decodeUtf8)
 import Data.Text.Lazy.IO as I (writeFile)
+import Options (Options (..), parseOptions)
 import Prettyprinter (defaultLayoutOptions, layoutPretty, pretty)
 import Prettyprinter.Render.String (renderString)
 import Spec.AuthorityToken qualified as AuthorityToken
@@ -20,7 +21,9 @@ import Prelude
 
 main :: IO ()
 main = do
-  I.writeFile "bench.csv" $
+  options <- parseOptions
+
+  I.writeFile options.output $
     (decodeUtf8 . encodeDefaultOrderedByName) res
 
   mapM_ (putStrLn . renderString . layoutPretty defaultLayoutOptions . pretty) res

agora-bench/Options.hs

@@ -0,0 +1,31 @@
+module Options (Options (..), parseOptions) where
+
+import Options.Applicative ((<**>))
+import Options.Applicative qualified as Opt
+
+newtype Options = Options
+  { output :: FilePath
+  }
+
+outputOpt :: Opt.Parser FilePath
+outputOpt =
+  Opt.strOption
+    ( Opt.long "output-path"
+        <> Opt.short 'o'
+        <> Opt.metavar "OUTPUT_PATH"
+        <> Opt.value "./bench.csv"
+        <> Opt.help "The path of the bench report file."
+    )
+
+benchOpt :: Opt.Parser Options
+benchOpt = Options <$> outputOpt
+
+parseOptions :: IO Options
+parseOptions = Opt.execParser p
+  where
+    p =
+      Opt.info
+        (benchOpt <**> Opt.helper)
+        ( Opt.fullDesc
+            <> Opt.progDesc "Generate benchmark report of agora scripts"
+        )

agora.cabal

@@ -212,13 +212,17 @@ benchmark agora-bench
   import:         lang, deps
   hs-source-dirs: agora-bench
   main-is:        Main.hs
-  other-modules:  Bench
+  other-modules:
+    Bench
+    Options
+
   type:           exitcode-stdio-1.0
   build-depends:
     , agora
     , agora-sample
     , agora-spec
     , cassava
+    , optparse-applicative
 
 executable agora-scripts
   import:         lang, deps, exe-opts

flake.nix

@@ -154,6 +154,26 @@
           mkdir $out
         '';
 
+      benchCheckFor = system: agora-bench:
+        let
+          pkgs = nixpkgsFor system;
+          pkgs' = nixpkgsFor' system;
+        in
+        pkgs.runCommand "bench-check"
+          {
+            bench = "${agora-bench}/bin/agora-bench";
+            nativeBuildInputs = [
+              pkgs'.diffutils
+            ];
+          } ''
+          export LC_CTYPE=C.UTF-8
+          export LC_ALL=C.UTF-8
+          export LANG=C.UTF-8
+          cd ${self}
+          make bench_check || (echo "    Please run 'make bench'" ; exit 1)
+          mkdir $out 
+        '';
+
     in
     {
       project = perSystem projectFor;
@@ -177,6 +197,7 @@
       checks = perSystem (system:
         self.flake.${system}.checks // {
           formatCheck = formatCheckFor system;
+          benchCheck = benchCheckFor system self.flake.${system}.packages."agora:bench:agora-bench";
           agora = self.flake.${system}.packages."agora:lib:agora";
           agora-test = self.flake.${system}.packages."agora:test:agora-test";
         });