From 9d169a44af9ee37829c174fb830d0082cfe03ea8 Mon Sep 17 00:00:00 2001
From: Unai Martinez-Corral <umartinezcorral@antmicro.com>
Date: Fri, 18 Mar 2022 04:09:00 +0100
Subject: [PATCH] add tests with pytest

Signed-off-by: Unai Martinez-Corral <umartinezcorral@antmicro.com>
---
 .github/workflows/Pipeline.yml |  7 ++----
 test/requirements.txt          |  1 +
 test/wrappers.py               | 46 ++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 5 deletions(-)
 create mode 100644 test/requirements.txt
 create mode 100644 test/wrappers.py

diff --git a/.github/workflows/Pipeline.yml b/.github/workflows/Pipeline.yml
index 6cc03a2..396dcbc 100644
--- a/.github/workflows/Pipeline.yml
+++ b/.github/workflows/Pipeline.yml
@@ -214,8 +214,5 @@ jobs:
         run: |
           . ./.github/scripts/activate.sh
 
-          for tool in place route synth write-fasm; do
-            echo "::group::Test f4pga-$tool"
-            "f4pga-$tool" || echo "Failing?"
-            echo "::endgroup::"
-          done;
+          pip3 install -r ./test/requirements.txt
+          pytest -vsrA --color=yes test/wrappers.py
diff --git a/test/requirements.txt b/test/requirements.txt
new file mode 100644
index 0000000..e079f8a
--- /dev/null
+++ b/test/requirements.txt
@@ -0,0 +1 @@
+pytest
diff --git a/test/wrappers.py b/test/wrappers.py
new file mode 100644
index 0000000..ea18079
--- /dev/null
+++ b/test/wrappers.py
@@ -0,0 +1,46 @@
+from pytest import mark
+from sys import stdout, stderr
+
+from subprocess import check_call
+
+
+@mark.xfail
+@mark.parametrize("wrapper", ['place', 'route', 'synth', 'write-fasm'])
+def test_wrapper(wrapper):
+    print(f"\n::group::Test {wrapper}")
+    stdout.flush()
+    stderr.flush()
+    try:
+        check_call(f"f4pga-{wrapper}")
+    finally:
+        print("\n::endgroup::")
+
+
+@mark.xfail
+@mark.parametrize(
+    "wrapper",
+    [
+        'symbiflow_generate_constraints',
+        'symbiflow_pack',
+        'symbiflow_place',
+        'symbiflow_route',
+        'symbiflow_synth',
+        'symbiflow_write_bitstream',
+        'symbiflow_write_fasm',
+        'symbiflow_write_xml_rr_graph',
+        'vpr_common',
+        'symbiflow_analysis',
+        'symbiflow_repack',
+        'symbiflow_generate_bitstream',
+        'symbiflow_generate_libfile',
+        'ql_symbiflow'
+    ]
+)
+def test_shell_wrapper(wrapper):
+    print(f"\n::group::Test {wrapper}")
+    stdout.flush()
+    stderr.flush()
+    try:
+        check_call(f"{wrapper}")
+    finally:
+        print("\n::endgroup::")