icecat: add release icecat-140.6.0-1gnu1 for ecne

icecat/toolkit/modules/subprocess/test/xpcshell/data_test_child.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+
+# This file is run by test_subprocess_child.js. To test this file in isolation:
+# python3 -u toolkit/modules/subprocess/test/xpcshell/data_test_child.py spawn_child_and_exit
+# Then separately, to the displayed URL "Listening at http://127.0.0.1:12345",
+# with 12345 being a random port,
+# curl http://127.0.0.1:12345 -X DELETE  # request exit of parent
+# curl http://127.0.0.1:12345            # request exit of child
+
+import sys
+import time
+
+
+def sleep_for_a_little_bit():
+    time.sleep(0.2)
+
+
+def spawn_child_and_exit(is_breakaway_job):
+    """
+    Spawns and exits child processes to allow tests to verify that they detect
+    specifically the exit of this (parent) process.
+
+    The expected sequence of outputs is as follows:
+    1. parent_start
+    2. first_child_start_and_exit
+    3. parent_after_first_child_exit
+    4. spawned_child_start
+    5. Listening at http://127.0.0.1:12345  - with 12345 being random port
+    6. child_received_http_request          - DELETE request from test.
+    7. data_from_child:kill_parent
+    8. parent_exit
+       ( now the parent has exit)
+       ( child_process_still_alive_1 response sent to request from step 6)
+       ( wait for new request from client to request child to exit )
+       ( child_process_still_alive_2 response sent to that new request )
+    9. spawned_child_exit
+    """
+    import subprocess
+
+    print("1. parent_start", flush=True)
+
+    # Start and exit a child process (used to make sure that we do not
+    # mistakenly detect an exited child process for the parent process).
+    subprocess.run(
+        [sys.executable, "-c", "print('2. first_child_start_and_exit')"],
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+    )
+    # Wait a bit to make sure that the child's exit signal has been processed.
+    # This is not strictly needed, because we don't expect the child to affect
+    # the parent, but in case of a flawed implementation, this would enable the
+    # test to detect a bad implementation (by observing the exit before the
+    # "parent_after_first_child_exit" message below).
+    sleep_for_a_little_bit()
+    print("3. parent_after_first_child_exit", flush=True)
+
+    creationflags = 0
+    if is_breakaway_job:
+        # See comment in test_subprocess_child.js; in short we need this flag
+        # to make sure that the child outlives the parent when the subprocess
+        # implementation calls TerminateJobObject.
+        creationflags = subprocess.CREATE_BREAKAWAY_FROM_JOB
+    child_proc = subprocess.Popen(
+        [sys.executable, "-u", __file__, "spawned_child"],
+        creationflags=creationflags,
+        # We don't need this pipe, but when this side of the process exits,
+        # the pipe is closed, which the child can use to detect that the parent
+        # has exit.
+        stdin=subprocess.PIPE,
+        # We are using stdout as a control channel to allow the child to
+        # notify us when the process is done.
+        stdout=subprocess.PIPE,
+        # stderr is redirected to the real stdout, so that the caller can
+        # still observe print() from spawned_child.
+        stderr=sys.stdout,
+    )
+
+    # This blocks until the child has notified us.
+    data_from_child = child_proc.stdout.readline().decode().rstrip()
+    print(f"7. data_from_child:{data_from_child}", flush=True)
+    print("8. parent_exit", flush=True)
+
+    # Wait a little bit to make sure that stdout has been flushed (and read by
+    # the caller) when the process exits.
+    sleep_for_a_little_bit()
+    sys.exit(0)
+
+
+def spawned_child():
+    import http.server
+    import socketserver
+
+    def print_to_parent_stdout(msg):
+        # The parent maps our stderr to its stdout.
+        print(msg, flush=True, file=sys.stderr)
+
+    # This is spawned via spawn_child_and_exit.
+    print_to_parent_stdout("4. spawned_child_start")
+
+    class RequestHandler(http.server.BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass  # Disable logging
+
+        def do_DELETE(self):
+            print_to_parent_stdout("6. child_received_http_request")
+            # Let the caller know that we are responsive.
+            self.send_response(200)
+            self.send_header("Connection", "close")
+            self.end_headers()
+
+            # Wait a little bit to allow the network request to be
+            # processed by the client. If for some reason the termination
+            # of the parent also kills the child, then at least the client
+            # has had a chance to become aware of it.
+            sleep_for_a_little_bit()
+
+            # Now ask the parent to exit, and continue here.
+            print("kill_parent", flush=True)
+            # When the parent exits, stdin closes, which we detect here:
+            res = sys.stdin.read(1)
+            if len(res):
+                print_to_parent_stdout("spawned_child_UNEXPECTED_STDIN")
+
+            # If we make it here, it means that this child outlived the
+            # parent, and we can let the client know.
+            # (if the child process is terminated prematurely, the client
+            # would also know through a disconnected socket).
+            self.wfile.write(b"child_process_still_alive_1")
+
+        def do_GET(self):
+            self.send_response(200)
+            self.send_header("Connection", "close")
+            self.end_headers()
+            self.wfile.write(b"child_process_still_alive_2")
+
+    # Starts a server that handles two requests and then closes the server.
+    with socketserver.TCPServer(("127.0.0.1", 0), RequestHandler) as server:
+        host, port = server.server_address[:2]
+        print_to_parent_stdout(f"5. Listening at http://{host}:{port}")
+
+        # Expecting DELETE request (do_DELETE)
+        server.handle_request()
+
+        # Expecting GET request (do_GET)
+        server.handle_request()
+
+    print_to_parent_stdout("9. spawned_child_exit")
+    sys.exit(0)
+
+
+cmd = sys.argv[1]
+if cmd == "spawn_child_and_exit":
+    spawn_child_and_exit(is_breakaway_job=False)
+elif cmd == "spawn_child_in_breakaway_job_and_exit":
+    spawn_child_and_exit(is_breakaway_job=True)
+elif cmd == "spawned_child":
+    spawned_child()
+else:
+    raise Exception(f"Unknown command: {cmd}")

icecat/toolkit/modules/subprocess/test/xpcshell/data_test_script.py

@@ -43,6 +43,14 @@ elif cmd == "pwd":
 elif cmd == "print_args":
     for arg in sys.argv[2:]:
         output(arg)
+elif cmd == "print_python_executable_path":
+    import psutil  # not stdlib, put already part of our virtualenv.
+
+    # This returns the final executable that launched this script.
+    # We cannot use sys.executable because on Windows that would point to the
+    # python.exe parent that spawns this process.
+    # See getRealPythonExecutable in test_subprocess.js.
+    output(psutil.Process().exe())
 elif cmd == "ignore_sigterm":
     signal.signal(signal.SIGTERM, signal.SIG_IGN)
 
@@ -54,6 +62,17 @@ elif cmd == "ignore_sigterm":
             import time
 
             time.sleep(3600)
+elif cmd == "close_stdin_and_wait_forever":
+    # Do NOT use non-stdlib modules here; this runs outside our virtualenv,
+    # via the program identified by print_python_executable_path.
+    os.close(sys.stdin.fileno())
+    output("stdin_closed")
+    while True:
+        pass  # Test should kill the program
+elif cmd == "close_pipes_and_wait_for_stdin":
+    os.close(sys.stdout.fileno())
+    os.close(sys.stderr.fileno())
+    sys.stdin.buffer.read(1)
 elif cmd == "print":
     output(sys.argv[2], stream=sys.stdout, print_only=True)
     output(sys.argv[3], stream=sys.stderr, print_only=True)

icecat/toolkit/modules/subprocess/test/xpcshell/head.js

@@ -11,3 +11,32 @@ const { AppConstants } = ChromeUtils.importESModule(
 ChromeUtils.defineESModuleGetters(this, {
   Subprocess: "resource://gre/modules/Subprocess.sys.mjs",
 });
+
+// Given the full path to a python executable (pathToPython), this function
+// looks up the ultimate "real" Python program that runs the Python script.
+// By default, pathToPython runs Python in a virtualenv, which on Windows
+// results in two python.exe being spawned: python.exe (at pathToPython) spawns
+// a new python.exe (outside the virtualenv, but configured by the parent
+// python.exe to use the virtualenv modules).
+//
+// In test_subprocess_stdin_closed_by_program test below in test_subprocess.js,
+// we run a script that closes stdin. This closed stdin does not propagate back
+// via the parent python.exe, so to make sure that it does, we directly launch
+// the real Python interpreter. This runs outside the virtualenv.
+//
+// Similarly, test_subprocess_stdout_closed_by_program in test_subprocess.js
+// runs a script that closes stdout, which only propagates back to the program
+// when the real Python interpreter is launched instead of a wrapper.
+async function getRealPythonExecutable(pathToPython) {
+  const TEST_SCRIPT = do_get_file("data_test_script.py").path;
+  let proc = await Subprocess.call({
+    command: pathToPython,
+    arguments: ["-u", TEST_SCRIPT, "print_python_executable_path"],
+  });
+  let { exitCode } = await proc.wait();
+  equal(exitCode, 0, "Successfully got executable path");
+  let count = await proc.stdout.readUint32();
+  let realPythonPath = await proc.stdout.readString(count);
+  ok(realPythonPath, `Found path to Python program: ${realPythonPath}`);
+  return realPythonPath;
+}

icecat/toolkit/modules/subprocess/test/xpcshell/test_subprocess.js

@@ -36,7 +36,7 @@ let readAll = async function (pipe) {
   return result.join("");
 };
 
-add_task(async function setup() {
+add_setup(async function setup() {
   PYTHON = await Subprocess.pathSearch(Services.env.get("PYTHON"));
 
   PYTHON_BIN = PathUtils.filename(PYTHON);
@@ -421,6 +421,79 @@ add_task(async function test_subprocess_eof() {
   equal(exitCode, 0, "Got expected exit code");
 });
 
+// Regression test for bug 1983138.
+add_task(async function test_subprocess_stdin_closed_by_program() {
+  // On Windows, the actual python.exe that runs the script is a child of the
+  // python.exe of the virtualenv, and closing stdin from that child does not
+  // propagate back, which would cause the proc.stdin.write() call below to
+  // succeed unexpectedly.
+  //
+  // To avoid this issue, use the real Python, see getRealPythonExecutable.
+  let proc = await Subprocess.call({
+    command: await getRealPythonExecutable(PYTHON),
+    arguments: ["-u", TEST_SCRIPT, "close_stdin_and_wait_forever"],
+  });
+
+  info("Waiting for program to notify us via stdout after closing stdin");
+
+  equal(
+    await read(proc.stdout),
+    "stdin_closed",
+    "Spawned process closed stdin"
+  );
+
+  // This is the most interesting part - write() should reject with an error.
+  await Assert.rejects(
+    proc.stdin.write("a"),
+    function (e) {
+      equal(
+        e.errorCode,
+        Subprocess.ERROR_END_OF_FILE,
+        "Got the expected error code"
+      );
+      return /File closed/.test(e.message);
+    },
+    "Promise should be rejected after program closed stdin"
+  );
+
+  let { exitCode } = await proc.kill();
+
+  // On UNIX platforms, our subprocess.kill() implementation sends SIGTERM,
+  // which Python handles and exiting with -15. On Windows, we send SIGKILL
+  // which the program cannot even handle and the exit code is -9.
+  const expectedExitCode = AppConstants.platform == "win" ? -9 : -15;
+  equal(exitCode, expectedExitCode, "Got expected exit code");
+});
+
+// Regression test for bug 1983138.
+add_task(async function test_subprocess_stdout_closed_by_program() {
+  // On Windows, the actual python.exe that runs the script is a child of the
+  // python.exe of the virtualenv, and closing stdout from that child does not
+  // propagate back, which would cause the proc.stdout.readString() call (via
+  // read) below to never resolve.
+  //
+  // To avoid this issue, use the real Python, see getRealPythonExecutable.
+  let proc = await Subprocess.call({
+    command: await getRealPythonExecutable(PYTHON),
+    arguments: ["-u", TEST_SCRIPT, "close_pipes_and_wait_for_stdin"],
+  });
+
+  // This is the most interesting part - readString() should resolve.
+  equal(
+    await proc.stdout.readString(),
+    "",
+    "stdout read should resolve to empty string upon close"
+  );
+
+  let { exitCode } = await proc.kill();
+
+  // On UNIX platforms, our subprocess.kill() implementation sends SIGTERM,
+  // which Python handles and exiting with -15. On Windows, we send SIGKILL
+  // which the program cannot even handle and the exit code is -9.
+  const expectedExitCode = AppConstants.platform == "win" ? -9 : -15;
+  equal(exitCode, expectedExitCode, "Got expected exit code");
+});
+
 add_task(async function test_subprocess_invalid_json() {
   let proc = await Subprocess.call({
     command: PYTHON,

icecat/toolkit/modules/subprocess/test/xpcshell/test_subprocess_child.js

@@ -0,0 +1,203 @@
+"use strict";
+
+const { TestUtils } = ChromeUtils.importESModule(
+  "resource://testing-common/TestUtils.sys.mjs"
+);
+const { setTimeout } = ChromeUtils.importESModule(
+  "resource://gre/modules/Timer.sys.mjs"
+);
+
+let PYTHON;
+
+add_setup(async () => {
+  PYTHON = await Subprocess.pathSearch(Services.env.get("PYTHON"));
+});
+
+// Send a request to the test server. We expect a non-empty response. Any error
+// is caught and returned as an empty string.
+async function fetchResponseText(url, method) {
+  const timeoutCtrl = new AbortController();
+
+  // We expect the (localhost) server to respond soon. If the server is not
+  // listening, we will eventually time out. Let's have a generous deadlines:
+  // - DELETE is given plenty of time, because the request is expected to
+  //   always be accepted, followed by IPC and process termination.
+  // - GET is given a short deadline, because we expect either an immediate
+  //   response (server is around) or no response (server was closed).
+  const timeout = method === "DELETE" ? 10000 : 1000;
+  // eslint-disable-next-line mozilla/no-arbitrary-setTimeout
+  setTimeout(() => timeoutCtrl.abort(), timeout);
+  // (^ AbortSignal.timeout() would be nicer but cannot be used, because it
+  //    throws: "The current global does not support timeout".)
+
+  try {
+    let res = await fetch(url, { method, signal: timeoutCtrl.signal });
+    return await res.text();
+  } catch (e) {
+    info(`fetch() request to kill parent failed: ${e}`);
+    if (
+      e.name !== "AbortError" &&
+      e.message !== "NetworkError when attempting to fetch resource."
+    ) {
+      ok(false, `Unexpected error: ${e}`);
+    }
+    return "";
+  }
+}
+
+async function do_test_spawn_parent_with_child(isBreakAwayJob) {
+  let testCmd;
+  if (isBreakAwayJob) {
+    Assert.equal(AppConstants.platform, "win", "Breakaway is Windows-only");
+    testCmd = "spawn_child_in_breakaway_job_and_exit";
+  } else {
+    testCmd = "spawn_child_and_exit";
+  }
+  const TEST_SCRIPT = do_get_file("data_test_child.py").path;
+  info(`Launching proc: ${PYTHON} -u ${TEST_SCRIPT} ${testCmd}`);
+  const proc = await Subprocess.call({
+    command: PYTHON,
+    arguments: ["-u", TEST_SCRIPT, testCmd],
+  });
+  const exitPromise = proc.wait();
+  let exited = false;
+  exitPromise.then(() => {
+    exited = true;
+  });
+
+  info(`Spawned process with pid ${proc.pid}, waiting for child`);
+
+  // We'll accumulate stdout, and reset what we've received so far after
+  // checking that the content matches with our expectations.
+  let stdoutText = "";
+  let stdoutDonePromise = (async () => {
+    let seenParentExit = false;
+    for (let s; (s = await proc.stdout.readString()); ) {
+      // On Windows, print() uses \r\n instead of \n. Drop it.
+      s = s.replaceAll("\r", "");
+      stdoutText += s;
+      dump(`Received stdout from test script: ${s}\n`);
+
+      seenParentExit ||= stdoutText.includes("parent_exit");
+      if (!seenParentExit) {
+        Assert.ok(!exited, "Process should not have exited yet");
+      }
+    }
+  })();
+
+  const EXPECTED_STDOUT_UNTIL_LISTENING = `\
+1. parent_start
+2. first_child_start_and_exit
+3. parent_after_first_child_exit
+4. spawned_child_start
+5. Listening at `; // followed by http://127.0.0.1:<digits>\n
+  const EXPECTED_STDOUT_AT_PARENT_EXIT = `\
+6. child_received_http_request
+7. data_from_child:kill_parent
+8. parent_exit
+`;
+  const EXPECTED_STDOUT_AT_CHILD_EXIT = `\
+9. spawned_child_exit
+`;
+
+  await TestUtils.waitForCondition(
+    () => stdoutText.startsWith(EXPECTED_STDOUT_UNTIL_LISTENING),
+    "Waiting for (parent) process to start child with listening HTTP server"
+  );
+
+  const url = stdoutText.replace(EXPECTED_STDOUT_UNTIL_LISTENING, "").trim();
+  ok(/^http:\/\/127\.0\.0\.1:\d+$/.test(url), `Found server URL: ${url}`);
+  stdoutText = ""; // Reset now that we have confirmed its content.
+
+  // When the server receives this request, it triggers exit of the process.
+  const promiseResponseToExitParent = fetchResponseText(url, "DELETE");
+
+  info("Waiting for spawned (parent) process to exit");
+  const { exitCode } = await exitPromise;
+  equal(exitCode, 0, "Got expected exit code");
+  equal(stdoutText, EXPECTED_STDOUT_AT_PARENT_EXIT, "stdout before exit");
+  stdoutText = ""; // Reset again after checking its content.
+
+  // Now two things can happen:
+  // - Either the child lives on, and we can send a request to it to ask the
+  //   child to exit as well.
+  // - Or the process tree rooted at the parent dies, and the child is taken
+  //   with it. The initial fetch() might succeed or reject. A new fetch() is
+  //   most likely going to reject.
+  // On Linux and macOS, the child appears to live on.
+  // On Windows, the whole process tree dies, when run under this test
+  //  (but somehow not when the Python script is run in isolation...?).
+
+  const responseToExitParent = await promiseResponseToExitParent;
+  if (!responseToExitParent) {
+    info("fetch() request to kill parent failed");
+  }
+
+  info("Checking whether the child process is still alive...");
+  const responseToChildAlive = await fetchResponseText(url, "GET");
+  const wasChildAlive = !!responseToChildAlive;
+
+  if (wasChildAlive) {
+    // Mainly a sanity check: If the second request gots through, then the
+    // first request should have received the expected response.
+    equal(responseToExitParent, "child_process_still_alive_1", "Still alive 1");
+    equal(responseToChildAlive, "child_process_still_alive_2", "Still alive 2");
+  } else {
+    info("fetch() request to check child liveness failed");
+  }
+
+  if (AppConstants.platform === "win" && !isBreakAwayJob) {
+    ok(!wasChildAlive, "Child process exits when the parent exits");
+  } else {
+    ok(wasChildAlive, "Child process outlives parent");
+  }
+
+  await stdoutDonePromise;
+
+  // On Windows, we close the pipes as soon as the parent process was detected
+  // to have exited. This prevents the child's write to be read.
+  if (wasChildAlive && AppConstants.platform !== "win") {
+    equal(stdoutText, EXPECTED_STDOUT_AT_CHILD_EXIT, "Stdout from child");
+  } else {
+    equal(stdoutText, "", "No more stdout after parent exited (with child)");
+  }
+}
+
+// Tests spawning a process that exits after spawning a child process, and
+// verify that the parent process is detected as exited, while the child
+// process is still running.
+add_task(async function test_spawn_child_outliving_parent_process() {
+  // Our subprocess implementation (subprocess_win.worker.js) puts the spawned
+  // process in a job, and cleans up that job upon process exit by calling
+  // TerminateJobObject. This causes all other (child) processes that are part
+  // of the job to terminate. To make sure that the child process outlives the
+  // parent, we have to launch it with the CREATE_BREAKAWAY_FROM_JOB flag.
+  const isBreakAwayJob = AppConstants.platform == "win";
+  await do_test_spawn_parent_with_child(isBreakAwayJob);
+});
+
+// On Windows, child processes are terminated along with the parent by default,
+// because our subprocess implementation puts the process in a Job, and the
+// TerminateJobObject call terminates all associated child processes. This
+// test confirms the default behavior, as opposed to what we see in
+// test_spawn_child_outliving_parent_process.
+add_task(
+  { skip_if: () => AppConstants.platform != "win" },
+  async function test_child_terminated_on_process_termination() {
+    await do_test_spawn_parent_with_child(false);
+  }
+);
+
+add_task(async function test_cleanup() {
+  let { getSubprocessImplForTest } = ChromeUtils.importESModule(
+    "resource://gre/modules/Subprocess.sys.mjs"
+  );
+
+  let worker = getSubprocessImplForTest().Process.getWorker();
+
+  let openFiles = await worker.call("getOpenFiles", []);
+  let processes = await worker.call("getProcesses", []);
+
+  equal(openFiles.size, 0, "No remaining open files");
+  equal(processes.size, 0, "No remaining processes");
+});

icecat/toolkit/modules/subprocess/test/xpcshell/test_subprocess_polling.js

@@ -0,0 +1,63 @@
+"use strict";
+
+const { getSubprocessImplForTest } = ChromeUtils.importESModule(
+  "resource://gre/modules/Subprocess.sys.mjs"
+);
+
+let PYTHON;
+const TEST_SCRIPT = do_get_file("data_test_script.py").path;
+
+add_setup(async () => {
+  PYTHON = await Subprocess.pathSearch(Services.env.get("PYTHON"));
+});
+
+// When the last process exits, we should stop polling.
+// This is a regression test for bug 1982950
+add_task(async function test_polling_only_when_process_is_running() {
+  let worker = getSubprocessImplForTest().Process.getWorker();
+
+  equal(
+    await worker.call("getIsPolling", []),
+    false,
+    "Initially not polling before starting a program"
+  );
+
+  let proc = await Subprocess.call({
+    command: await getRealPythonExecutable(PYTHON),
+    arguments: ["-u", TEST_SCRIPT, "close_pipes_and_wait_for_stdin"],
+  });
+
+  equal(
+    await worker.call("getIsPolling", []),
+    true,
+    "Is polling while process is active"
+  );
+
+  // Make sure that we have completed reading before exiting the program,
+  // to rule out pending polls from open stdout pipe. Note that we expect
+  // to be closed because the program closed stdout prematurely.
+  equal(
+    await proc.stdout.readString(),
+    "",
+    "Test program should have closed stdout prematurely without stdout"
+  );
+
+  equal(
+    await worker.call("getIsPolling", []),
+    true,
+    "Is still polling while process is active"
+  );
+
+  info("Closing stdin to trigger exit");
+  await proc.stdin.close();
+
+  let { exitCode } = await proc.wait();
+  equal(exitCode, 0, "Got expected exit code");
+
+  // This part is the regression test for bug 1982950.
+  equal(
+    await worker.call("getIsPolling", []),
+    false,
+    "Not polling when last process has exited"
+  );
+});

icecat/toolkit/modules/subprocess/test/xpcshell/xpcshell.toml

@@ -5,6 +5,7 @@ skip-if = ["os == 'android'"]
 subprocess = true
 support-files = [
   "data_text_file.txt",
+  "data_test_child.py",
   "data_test_script.py",
 ]
 
@@ -15,6 +16,8 @@ skip-if = [
 ]
 run-sequentially = "very high failure rate in parallel"
 
+["test_subprocess_child.js"]
+
 ["test_subprocess_connectRunning.js"]
 skip-if = [
   "os == 'mac' && os_version == '11.20' && arch == 'aarch64'", # Bug 1936343
@@ -24,3 +27,5 @@ skip-if = [
 ["test_subprocess_getEnvironment.js"]
 
 ["test_subprocess_pathSearch.js"]
+
+["test_subprocess_polling.js"]