Add a release test script

Confirms that all the examples that aren't tested in the test suite at
least run. Easy to miss brokenness there. Output can be inspected
manually (it'll vary depending on the kernel version).

Fix defconfig_oldconfig.py, which hadn't been properly updated for the
new API.

4 files changed, 64 insertions, 11 deletions

diff --git a/examples/defconfig_oldconfig.py b/examples/defconfig_oldconfig.py
index 8e72c8a..84aa134 100644
--- a/examples/defconfig_oldconfig.py
+++ b/examples/defconfig_oldconfig.py
@@ -15,23 +15,23 @@
 import kconfiglib
 import sys
 
-conf = kconfiglib.Kconfig(sys.argv[1])
+kconf = kconfiglib.Kconfig(sys.argv[1])
 
 # Mirrors defconfig
-conf.load_config("arch/x86/configs/x86_64_defconfig")
-conf.write_config(".config")
+kconf.load_config("arch/x86/configs/x86_64_defconfig")
+kconf.write_config(".config")
 
 # Mirrors the first oldconfig
-conf.load_config(".config")
-conf.syms["ETHERNET"].set_value(0)
-conf.write_config(".config")
+kconf.load_config(".config")
+kconf.syms["ETHERNET"].set_value(0)
+kconf.write_config(".config")
 
 # Mirrors the second oldconfig
-conf.load_config(".config")
-conf.syms["ETHERNET"].set_value(2)
-for s in conf:
+kconf.load_config(".config")
+kconf.syms["ETHERNET"].set_value(2)
+for s in kconf.defined_syms:
     if s.user_value is None and 0 in s.assignable:
         s.set_value(0)
 
 # Write the final configuration
-conf.write_config(".config")
+kconf.write_config(".config")
diff --git a/examples/print_sym_info.py b/examples/print_sym_info.py
index 120bc07..3ee3c97 100644
--- a/examples/print_sym_info.py
+++ b/examples/print_sym_info.py
@@ -4,7 +4,7 @@
 #
 #   $ make [ARCH=<arch>] scriptconfig SCRIPT=Kconfiglib/examples/print_sym_info.py SCRIPT_ARG=<name>
 #
-# Example output for SCRIPT_ARG=modules:
+# Example output for SCRIPT_ARG=MODULES:
 #
 # menuconfig MODULES
 # 	bool
diff --git a/tests/reltest b/tests/reltest
new file mode 100755
index 0000000..8a3d581
--- /dev/null
+++ b/tests/reltest
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Runs the test suite and all examples scripts with Python 2 and Python 3,
+# bailing immediately if anything fails. For the examples that aren't tested in
+# the test suite, we just confirm that they at least run.
+#
+# Should be run from the kernel root with  $ Kconfiglib/tests/reltest
+
+test_script() {
+    echo -e "\n================= $1 with $py =================\n"
+    if (( $# == 1)); then
+        make scriptconfig PYTHONCMD=$py SCRIPT=$1
+    else
+        make scriptconfig PYTHONCMD=$py SCRIPT=$1 SCRIPT_ARG="$2"
+    fi
+
+    if (($?)); then
+        echo "$1 failed to run with $py"
+        exit 1
+    fi
+}
+
+for py in python2 python3; do
+    echo -e "\n================= Test suite with $py =================\n"
+
+    if ! $py Kconfiglib/testsuite.py speedy; then
+        echo "test suite failed for $py"
+        exit 1
+    fi
+
+    # Check that the example scripts that aren't tested by the test suite run
+    # at least
+
+    test_script Kconfiglib/examples/defconfig_oldconfig.py
+    test_script Kconfiglib/examples/defconfig.py
+    test_script Kconfiglib/examples/eval_expr.py MODULES
+    test_script Kconfiglib/examples/find_symbol.py X86
+    test_script Kconfiglib/examples/help_grep.py general
+    test_script Kconfiglib/examples/print_sym_info.py MODULES
+    test_script Kconfiglib/examples/print_tree.py
+
+    $py Kconfiglib/examples/menuconfig.py Kconfiglib/examples/Kmenuconfig <<END
+BOOL
+n
+END
+    if (($?)); then
+        echo "menuconfig.py failed with $py"
+        exit 1
+    fi
+done
+
+echo "everything okay"
diff --git a/testsuite.py b/testsuite.py
index 90efc3d..8ba0edc 100644
--- a/testsuite.py
+++ b/testsuite.py
@@ -1705,6 +1705,7 @@ def run_compatibility_tests():
         print("{} arch/defconfig pairs tested".format(nconfigs))
     else:
         print("Some tests failed")
+        sys.exit(1)
 
 def get_arch_srcarch_list():
     """