Make Bytecode DSL the default

ci.jsonnet

@@ -75,16 +75,17 @@
     local bench_task(bench=null, benchmarks=BENCHMARKS) = super.bench_task(bench=bench, benchmarks=benchmarks),
     local bisect_bench_task     = self.bisect_bench_task,
 
-    local bytecode_dsl_env = task_spec({
+    // Manual interpreter variants (DSL disabled)
+    local manual_interpreter_env = task_spec({
         environment +: {
-            BYTECODE_DSL_INTERPRETER: "true"
+            BYTECODE_DSL_INTERPRETER: "false"
         },
     }),
-    local bytecode_dsl_gate(name) = bytecode_dsl_env + task_spec({
+    local manual_interpreter_gate(name) = manual_interpreter_env + task_spec({
         tags :: name,
     }),
-    local bytecode_dsl_bench = bytecode_dsl_env + task_spec({
-        name_suffix +:: ["bytecode-dsl"],
+    local manual_interpreter_bench = manual_interpreter_env + task_spec({
+        name_suffix +:: ["manual-interpreter"],
     }),
 
     // -----------------------------------------------------------------------------------------------------------------
@@ -116,7 +117,7 @@
             "darwin:aarch64:jdk-latest"  : tier3                     + provide(GPY_JVM_STANDALONE),
             "windows:amd64:jdk-latest"   : tier3                     + provide(GPY_JVM_STANDALONE),
         }),
-        "python-unittest-bytecode-dsl": gpgate + platform_spec(no_jobs) + bytecode_dsl_gate("python-unittest") + platform_spec({
+        "python-unittest-manual-interpreter": gpgate + platform_spec(no_jobs) + manual_interpreter_gate("python-unittest") + platform_spec({
             "linux:amd64:jdk-latest"     : daily     + t("01:00:00"),
             "linux:aarch64:jdk-latest"   : daily     + t("01:00:00"),
             "darwin:aarch64:jdk-latest"  : daily     + t("01:00:00"),
@@ -170,7 +171,7 @@
             "darwin:aarch64:jdk-latest"  : tier3                      + require(GRAAL_JDK_LATEST),
             "windows:amd64:jdk-latest"   : tier3                      + require(GRAAL_JDK_LATEST),
         }),
-        "python-junit-bytecode-dsl": gpgate + platform_spec(no_jobs) + bytecode_dsl_gate("python-junit") + platform_spec({
+        "python-junit-manual-interpreter": gpgate + platform_spec(no_jobs) + manual_interpreter_gate("python-junit") + platform_spec({
             "linux:amd64:jdk-latest"     : tier3                      + require(GRAAL_JDK_LATEST),
         }),
         "python-junit-maven": gpgate_maven + platform_spec(no_jobs) + platform_spec({
@@ -204,12 +205,12 @@
                 ],
             }),
         }),
-        "python-pgo-profile-bytecode-dsl": gpgate_ee + bytecode_dsl_env + platform_spec(no_jobs) + platform_spec({
+        "python-pgo-profile-manual-interpreter": gpgate_ee + manual_interpreter_env + platform_spec(no_jobs) + platform_spec({
             "linux:amd64:jdk-latest"     : post_merge + t("01:30:00") + task_spec({
                 run: [["mx", "python-native-pgo"]],
                 logs+: [
-                    "default-bytecode-dsl.iprof.gz",
-                    "default-bytecode-dsl.lcov",
+                    "default-manual-interpreter.iprof.gz",
+                    "default-manual-interpreter.lcov",
                 ],
             }),
         }),
@@ -219,7 +220,7 @@
             "darwin:aarch64:jdk-latest"  : tier3                     + require(GPY_NATIVE_STANDALONE),
             "windows:amd64:jdk-latest"   : tier3                     + require(GPY_NATIVE_STANDALONE) + batches(2),
         }),
-        "python-svm-unittest-bytecode-dsl": gpgate + platform_spec(no_jobs) + bytecode_dsl_gate("python-svm-unittest") + platform_spec({
+        "python-svm-unittest-manual-interpreter": gpgate + platform_spec(no_jobs) + manual_interpreter_gate("python-svm-unittest") + platform_spec({
             "linux:amd64:jdk-latest"     : tier2                     + provide(GPY_NATIVE_BYTECODE_DSL_STANDALONE),
         }),
         "python-tagged-unittest": gpgate + require(GPY_NATIVE_STANDALONE) + batches(TAGGED_UNITTESTS_SPLIT) + platform_spec(no_jobs) + platform_spec({
@@ -228,7 +229,7 @@
             "darwin:aarch64:jdk-latest"  : tier3,
             "windows:amd64:jdk-latest"   : daily     + t("02:00:00"),
         }),
-        "python-tagged-unittest-bytecode-dsl": gpgate + require(GPY_NATIVE_BYTECODE_DSL_STANDALONE) + batches(TAGGED_UNITTESTS_SPLIT) + bytecode_dsl_gate("python-tagged-unittest") + platform_spec(no_jobs) + platform_spec({
+        "python-tagged-unittest-manual-interpreter": gpgate + require(GPY_NATIVE_BYTECODE_DSL_STANDALONE) + batches(TAGGED_UNITTESTS_SPLIT) + manual_interpreter_gate("python-tagged-unittest") + platform_spec(no_jobs) + platform_spec({
             "linux:amd64:jdk-latest"     : tier3,
         }),
         "python-graalvm": gpgate + platform_spec(no_jobs) + platform_spec({
@@ -288,7 +289,7 @@
     },
 
     local need_pgo = task_spec({runAfter: ["python-pgo-profile-post_merge-linux-amd64-jdk-latest"]}),
-    local need_bc_pgo = task_spec({runAfter: ["python-pgo-profile-bytecode-dsl-post_merge-linux-amd64-jdk-latest"]}),
+    local need_bc_pgo = task_spec({runAfter: ["python-pgo-profile-manual-interpreter-post_merge-linux-amd64-jdk-latest"]}),
     local forks_warmup = forks("./mx.graalpython/warmup-fork-counts.json"),
     local forks_meso = forks("meso.json"),
     local raw_results = task_spec({
@@ -316,7 +317,7 @@
         }),
         for bench in ["micro", "meso", "macro"]
     } + {
-        [bench + "-bytecode-dsl"]: bench_task(bench) + bytecode_dsl_bench + platform_spec(no_jobs) + bench_variants({
+        [bench + "-manual-interpreter"]: bench_task(bench) + manual_interpreter_bench + platform_spec(no_jobs) + bench_variants({
             "vm_name:graalvm_ee_default_bc_dsl"                                : {"linux:amd64:jdk-latest" : daily      + t("08:00:00")},
             "vm_name:graalpython_enterprise_bc_dsl"                            : {"linux:amd64:jdk-latest" : daily      + t("08:00:00")},
         }),
@@ -351,7 +352,7 @@
         }),
         for bench in ["micro_small", "meso_small"]
     } + {
-        [bench + "-bytecode-dsl"]: bench_task(bench) + bytecode_dsl_bench + platform_spec(no_jobs) + bench_variants({
+        [bench + "-manual-interpreter"]: bench_task(bench) + manual_interpreter_bench + platform_spec(no_jobs) + bench_variants({
             "vm_name:graalvm_ee_default_interpreter_bc_dsl"                    : {"linux:amd64:jdk-latest" : daily     + t("04:00:00")},
             "vm_name:graalpython_enterprise_interpreter_bc_dsl"                : {"linux:amd64:jdk-latest" : weekly    + t("04:00:00")},
         }),
@@ -384,7 +385,7 @@
         }),
         for bench in ["warmup"]
     } + {
-        [bench + "-bytecode-dsl"]: bench_task(bench) + bytecode_dsl_bench + platform_spec(no_jobs) + bench_variants({
+        [bench + "-manual-interpreter"]: bench_task(bench) + manual_interpreter_bench + platform_spec(no_jobs) + bench_variants({
             "vm_name:graalvm_ee_default_bc_dsl"                                : {"linux:amd64:jdk-latest" : on_demand     + t("05:00:00") + forks_warmup},
             "vm_name:graalpython_enterprise_bc_dsl"                            : {"linux:amd64:jdk-latest" : on_demand     + t("05:00:00") + forks_warmup},
         }),
@@ -397,7 +398,7 @@
         }),
         for bench in ["heap", "micro_small_heap"]
     } + {
-        [bench + "-bytecode-dsl"]: bench_task(bench) + bytecode_dsl_bench + platform_spec(no_jobs) + bench_variants({
+        [bench + "-manual-interpreter"]: bench_task(bench) + manual_interpreter_bench + platform_spec(no_jobs) + bench_variants({
             "vm_name:graalvm_ee_default_interpreter_bc_dsl"             : {"linux:amd64:jdk-latest" : post_merge     + t("02:00:00") + need_bc_pgo},
             "vm_name:graalpython_enterprise_interpreter_bc_dsl"         : {"linux:amd64:jdk-latest" : weekly         + t("02:00:00")},
         }),
@@ -429,7 +430,7 @@
         for bench in ["pyperformance"]
     } + {
         // Bytecode DSL benchmarks with community benchmark suites for external numbers
-        [bench + "-bytecode-dsl"]: bench_task(bench, PY_BENCHMARKS) + bytecode_dsl_bench + platform_spec(no_jobs) + raw_results + bench_variants({
+        [bench + "-manual-interpreter"]: bench_task(bench, PY_BENCHMARKS) + manual_interpreter_bench + platform_spec(no_jobs) + raw_results + bench_variants({
             "vm_name:graalvm_ee_default_bc_dsl"                                : {"linux:amd64:jdk-latest" : weekly     + t("08:00:00")},
         }),
         for bench in ["pyperformance"]

ci/python-gate.libsonnet

@@ -158,6 +158,8 @@
         windows: {
             common: {
                 PATH: "$MAVEN_HOME\\bin;$PATH",
+                # Gradle: this feature doesn't work on all Windows CI machines
+                GRADLE_OPTS: "-Dorg.gradle.vfs.watch=false",
             },
             amd64: {},
             aarch64: {},
@@ -419,7 +421,7 @@
         environment+: $.environment(self.os, self.arch),
         packages+: $.packages(self.os, self.arch),
         run+: [
-            ["mx"] + self.mx_parameters + self.dy + self.primary_suite + [
+            ["mx", "--J", "@-Dtck.inlineVerifierInstrument=false"] + self.mx_parameters + self.dy + self.primary_suite + [
                 "--strict-compliance", "--primary", "gate", "--tags", self.tags, "-B=--force-deprecation-as-warning",
             ] + self.all_suites + self.gate_parameters,
         ],

graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/SandboxPolicyUntrustedTest.java

@@ -69,7 +69,8 @@ private static Value run(String source) {
                         .option("sandbox.MaxHeapMemory", "800MB") //
                         .option("sandbox.MaxCPUTime", "10s") //
                         .option("sandbox.MaxASTDepth", "100") //
-                        .option("sandbox.MaxStackFrames", "10") //
+                        // encodings import during ctx init takes 19 frames and that's the deepest
+                        .option("sandbox.MaxStackFrames", "25") //
                         .option("sandbox.MaxThreads", "1") //
                         .option("sandbox.MaxOutputStreamSize", "1MB") //
                         .option("sandbox.MaxErrorStreamSize", "1MB") //

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java

@@ -40,6 +40,7 @@
  */
 package com.oracle.graal.python.test.debug;
 
+import static com.oracle.graal.python.test.integration.PythonTests.eval;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -58,6 +59,7 @@
 import org.graalvm.polyglot.Context;
 import org.graalvm.polyglot.Context.Builder;
 import org.graalvm.polyglot.Source;
+import org.graalvm.polyglot.Value;
 import org.junit.After;
 import org.junit.Assume;
 import org.junit.Before;
@@ -93,7 +95,9 @@ public void dispose() {
 
     @Test
     public void testSteppingAsExpected() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         // test that various elements step as expected, including generators, statement level atomic
         // expressions, and roots
         final Source source = Source.newBuilder("python", "" +
@@ -186,7 +190,9 @@ public void testSteppingAsExpected() throws Throwable {
 
     @Test
     public void testException() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         final Source source = Source.newBuilder("python", "" +
                         "try:\n" +
                         "  1 / 0\n" +
@@ -210,7 +216,9 @@ public void testException() throws Throwable {
 
     @Test
     public void testInlineEvaluation() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         final Source source = Source.newBuilder("python", "" +
                         "y = 4\n" +
                         "def foo(x):\n" +
@@ -245,7 +253,9 @@ public void testInlineEvaluation() throws Throwable {
     @Test
     @SuppressWarnings("try")
     public void testBreakpointBuiltin() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         final Source source = Source.newBuilder("python", "" +
                         "def foo():\n" +
                         "  a = 1\n" +
@@ -267,7 +277,9 @@ public void testBreakpointBuiltin() throws Throwable {
 
     @Test
     public void testConditionalBreakpointInFunction() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         final Source source = Source.newBuilder("python", "" +
                         "def fun():\n" +
                         "  def prod(n):\n" +
@@ -313,7 +325,9 @@ public void testConditionalBreakpointInFunction() throws Throwable {
 
     @Test
     public void testConditionalBreakpointGlobal() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         final Source source = Source.newBuilder("python", "" +
                         "values = []\n" +
                         "for i in range(0, 10):\n" +
@@ -337,7 +351,9 @@ public void testConditionalBreakpointGlobal() throws Throwable {
 
     @Test
     public void testReenterArgumentsAndValues() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         // Test that after a re-enter, arguments are kept and variables are cleared.
         final Source source = Source.newBuilder("python", "" +
                         "def main():\n" +
@@ -392,7 +408,9 @@ public void testReenterArgumentsAndValues() throws Throwable {
     @Test
     @SuppressWarnings("deprecation")
     public void testGettersSetters() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         final Source source = Source.newBuilder("python", "" +
                         "class GetterOnly:\n" +
                         "  def __get__(self):\n" +
@@ -462,7 +480,9 @@ public void testGettersSetters() throws Throwable {
 
     @Test
     public void testInspectJavaArray() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         final Source source = Source.newBuilder("python", "" +
                         "import java\n" +
                         "a_int = java.type('int[]')(3)\n" +
@@ -508,7 +528,9 @@ public void testInspectJavaArray() throws Throwable {
 
     @Test
     public void testSourceFileURI() throws Throwable {
-        Assume.assumeFalse("TODO: debugger tests are broken on Bytecode DSL", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        Value isBytecodeDLS = eval("__graalpython__.is_bytecode_dsl_interpreter");
+        // GR-71618
+        Assume.assumeFalse("TODO: wrong stacktrace", isBytecodeDLS.asBoolean());
         if (System.getProperty("os.name").toLowerCase().contains("mac")) {
             // on the mac machines we run with symlinked directories and such and it's annoying to
             // cater for that

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -956,10 +956,10 @@ def parse_config(cls, config_path: Path):
             tags_dir = None
             if config_tags_dir := settings.get('tags_dir'):
                 tags_dir = (config_path.parent / config_tags_dir).resolve()
-            # Temporary hack for Bytecode DSL development in master branch:
+            # Temporary hack for the period when we support both Bytecode DSL and manual interpreter
             # noinspection PyUnresolvedReferences
-            if IS_GRAALPY and getattr(__graalpython__, 'is_bytecode_dsl_interpreter', False) and tags_dir:
-                new_tags_dir = (config_path.parent / (config_tags_dir + '_bytecode_dsl')).resolve()
+            if IS_GRAALPY and not getattr(__graalpython__, 'is_bytecode_dsl_interpreter', False) and tags_dir:
+                new_tags_dir = (config_path.parent / (config_tags_dir + '_manual_bci')).resolve()
                 if new_tags_dir.exists():
                     tags_dir = new_tags_dir
             return cls(

graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py

@@ -46,6 +46,23 @@
 is_enabled = 'ENABLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_STANDALONE_UNITTESTS'] == "true"
 constraints_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'constraints.txt')
 
+class TemporaryTestDirectory():
+    def __init__(self):
+        if 'STANDALONE_UNITTESTS_NO_CLEAN' in os.environ:
+            self.ctx = None
+            self.name = tempfile.mkdtemp()
+            print(f"Running test in {self.name}")
+        else:
+            self.ctx = tempfile.TemporaryDirectory()
+            self.name = self.ctx.name
+
+    def __enter__(self):
+        return self.name
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.ctx:
+            self.ctx.__exit__(exc_type, exc_val, exc_tb)
+
 def create_test_env():
     env = os.environ.copy()
     env["MVN_GRAALPY_VERSION"] = util.get_graalvm_version()
@@ -59,7 +76,7 @@ def test_native_executable_one_file():
         return
 
     env = create_test_env()
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with TemporaryTestDirectory() as tmpdir:
 
         source_file = os.path.join(tmpdir, "hello.py")
         with open(source_file, 'w') as f:
@@ -88,7 +105,7 @@ def test_native_executable_venv_and_one_file():
         return
 
     env = create_test_env()
-    with tempfile.TemporaryDirectory() as target_dir:
+    with TemporaryTestDirectory() as target_dir:
         source_file = os.path.join(target_dir, "hello.py")
         with open(source_file, 'w') as f:
             f.write("from termcolor import colored, cprint\n")
@@ -130,7 +147,7 @@ def test_native_executable_module():
         return
 
     env = create_test_env()
-    with tempfile.TemporaryDirectory() as tmp_dir:
+    with TemporaryTestDirectory() as tmp_dir:
 
         module_dir = os.path.join(tmp_dir, "hello_app")
         os.makedirs(module_dir, exist_ok=True)