From 8710723ee7ef7c7b296e8c5fd669c92d3c183588 Mon Sep 17 00:00:00 2001 From: "Jip J. Dekker" Date: Fri, 12 Jul 2024 10:20:26 +1000 Subject: [PATCH 01/14] Add support for the experimental MiniZinc black-box propagator interface --- CMakeLists.txt | 7 + Makefile.in | 4 +- changelog.in | 13 + cmake/GecodeSources.cmake | 1 + gecode/flatzinc/blackbox.cpp | 562 ++++++++++++++++++ gecode/flatzinc/blackbox.hh | 400 +++++++++++++ .../experimental/blackbox/fzn_blackbox.mzn | 13 + .../blackbox/fzn_blackbox_bounds.mzn | 11 + gecode/flatzinc/registry.cpp | 86 +++ 9 files changed, 1095 insertions(+), 2 deletions(-) create mode 100644 gecode/flatzinc/blackbox.cpp create mode 100644 gecode/flatzinc/blackbox.hh create mode 100644 gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn create mode 100644 gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn diff --git a/CMakeLists.txt b/CMakeLists.txt index ab27c0d4cc..66591a5881 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1285,6 +1285,13 @@ if(GECODE_ENABLE_FLATZINC) endif() endforeach() endif() + if(CMAKE_DL_LIBS) + foreach(kind shared static) + if(TARGET gecodeflatzinc_${kind}) + target_link_libraries(gecodeflatzinc_${kind} PUBLIC ${CMAKE_DL_LIBS}) + endif() + endforeach() + endif() endif() # Compatibility aggregate target for downstream projects expecting Gecode::gecode. diff --git a/Makefile.in b/Makefile.in index dbec331bc1..cb30b9b5ed 100755 --- a/Makefile.in +++ b/Makefile.in @@ -820,11 +820,11 @@ endif # FLATZINC # -FLATZINCSRC0 = flatzinc.cpp registry.cpp branch.cpp +FLATZINCSRC0 = flatzinc.cpp registry.cpp branch.cpp blackbox.cpp FLATZINC_GENSRC0 = parser.tab.cpp lexer.yy.cpp FLATZINCHDR0 = ast.hh conexpr.hh option.hh parser.hh \ plugin.hh registry.hh symboltable.hh varspec.hh \ - branch.hh branch.hpp lastval.hh complete.hh + branch.hh branch.hpp lastval.hh complete.hh blackbox.hh FLATZINCSRC = $(FLATZINCSRC0:%=gecode/flatzinc/%) FLATZINC_GENSRC = $(FLATZINC_GENSRC0:%=gecode/flatzinc/%) diff --git a/changelog.in b/changelog.in index d468a31620..be3ec50ac7 100755 --- a/changelog.in +++ b/changelog.in @@ -75,6 +75,19 @@ This release modernizes the Gecode build infrastructure, adds a first-class CMake package for downstream consumers, refreshes the autoconf build path, and updates CI coverage for current platforms. +[ENTRY] +Module: flatzinc +What: new +Rank: minor +[DESCRIPTION] +Add support for the experimental MiniZinc black-box propagator interface. A +FlatZinc model can request propagation using an external function, implemented +either as a shared library or as a subprocess, through two generic propagators: +gecode_blackbox (value propagation, scheduled once all inputs are fixed) and +gecode_blackbox_bounds (bounds propagation, scheduled on bound changes).The +blackbox_exec and blackbox_dll annotations select the execution mode and pass +through extra arguments. + [ENTRY] Module: minimodel What: bug diff --git a/cmake/GecodeSources.cmake b/cmake/GecodeSources.cmake index eb1809eba8..affe090e31 100644 --- a/cmake/GecodeSources.cmake +++ b/cmake/GecodeSources.cmake @@ -232,6 +232,7 @@ set(GECODE_GIST_SOURCES ) set(GECODE_FLATZINC_SOURCES + gecode/flatzinc/blackbox.cpp gecode/flatzinc/branch.cpp gecode/flatzinc/flatzinc.cpp gecode/flatzinc/lexer.yy.cpp diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp new file mode 100644 index 0000000000..7c662cd451 --- /dev/null +++ b/gecode/flatzinc/blackbox.cpp @@ -0,0 +1,562 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Copyright: + * Jip J. Dekker, 2026 + * + * This file is part of Gecode, the generic constraint + * development environment: + * http://www.gecode.org + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#include +#include +#endif + +namespace Gecode { +namespace FlatZinc { + +BlackBoxDLL::BlackBoxDLL(const std::string &name, + const std::vector &args) { + std::string loadError; +#ifdef _WIN32 + library = LoadLibraryA(name.c_str()); + if (!library) { + loadError = std::string("unable to locate library `") + name + "'"; + library = LoadLibraryA((std::string(name) + ".dll").c_str()); + } + if (!library) { + library = LoadLibraryA((std::string("lib") + name + ".dll").c_str()); + } +#else + library = dlopen(name.c_str(), RTLD_LAZY); + if (!library) { + loadError = std::string(dlerror()); + library = dlopen((name + ".so").c_str(), RTLD_NOW); + } + if (!library) { + library = dlopen((std::string("lib") + name + ".so").c_str(), RTLD_NOW); + } +#endif + if (!library) { + throw Error("Blackbox", "Unable to open dynamic library: " + loadError); + } + + // find symbol for blacbox function +#ifdef _WIN32 + dll_fzn_blackbox = reinterpret_cast( + GetProcAddress((HMODULE)library, "fzn_blackbox")); + std::string symError = "."; +#else + *(void **)(&dll_fzn_blackbox) = dlsym(library, "fzn_blackbox"); + std::string symError(": "); + if (!dll_fzn_blackbox) { + symError += std::string(dlerror()); + } +#endif + if (!dll_fzn_blackbox) { + throw Error("Blackbox", + "Unable to find symbol `fzn_blackbox` in dynamic library" + + symError); + } + + // Optionally call the initialisation function with the given arguments. It is + // not an error for the library to omit `fzn_initialize`. + void(__stdcall *dll_fzn_initialize)(const char **, size_t) = nullptr; +#ifdef _WIN32 + dll_fzn_initialize = reinterpret_cast( + GetProcAddress((HMODULE)library, "fzn_initialize")); +#else + *(void **)(&dll_fzn_initialize) = dlsym(library, "fzn_initialize"); +#endif + if (dll_fzn_initialize != nullptr) { + std::vector argv; + argv.reserve(args.size()); + for (const std::string &a : args) { + argv.push_back(a.c_str()); + } + dll_fzn_initialize(argv.data(), argv.size()); + } +} + +BlackBoxDLL::~BlackBoxDLL() { + if (library) { +#ifdef _WIN32 + FreeLibrary((HMODULE)library); +#else + dlclose(library); +#endif + } +} + +BlackBoxExec::BlackBoxExec(const std::string &program, + const std::vector &args) { +#ifdef _WIN32 + SECURITY_ATTRIBUTES saAttr; + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + + HANDLE g_hChildStd_IN_Rd = NULL; + HANDLE g_hChildStd_IN_Wr = NULL; + HANDLE g_hChildStd_OUT_Rd = NULL; + HANDLE g_hChildStd_OUT_Wr = NULL; + + // Create a pipe for the child process's STDOUT. + if (!CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0)) + std::cerr << "Stdout CreatePipe" << std::endl; + // Ensure the read handle to the pipe for STDOUT is not inherited. + if (!SetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0)) + std::cerr << "Stdout SetHandleInformation" << std::endl; + + // Create a pipe for the child process's STDIN + if (!CreatePipe(&g_hChildStd_IN_Rd, &g_hChildStd_IN_Wr, &saAttr, 0)) + std::cerr << "Stdin CreatePipe" << std::endl; + // Ensure the write handle to the pipe for STDIN is not inherited. + if (!SetHandleInformation(g_hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0)) + std::cerr << "Stdin SetHandleInformation" << std::endl; + + PROCESS_INFORMATION piProcInfo; + STARTUPINFOA siStartInfo; + + // Set up members of the PROCESS_INFORMATION structure. + ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); + + // Set up members of the STARTUPINFO structure. + // This structure specifies the STDIN and STDOUT handles for redirection. + ZeroMemory(&siStartInfo, sizeof(STARTUPINFOA)); + siStartInfo.cb = sizeof(STARTUPINFOA); + siStartInfo.hStdOutput = g_hChildStd_OUT_Wr; + siStartInfo.hStdInput = g_hChildStd_IN_Rd; + siStartInfo.dwFlags |= STARTF_USESTDHANDLES; + + // Build the command line: the program followed by the (quoted) arguments. + std::string prog = program; + for (const std::string &a : args) { + prog += " \""; + for (char ch : a) { + if (ch == '"' || ch == '\\') { + prog += '\\'; + } + prog += ch; + } + prog += '"'; + } + BOOL processStarted = + CreateProcessA(nullptr, + prog.data(), // command line + nullptr, // process security attributes + nullptr, // primary thread security attributes + TRUE, // handles are inherited + 0, // creation flags + nullptr, // use parent's environment + nullptr, // use parent's current directory + &siStartInfo, // STARTUPINFO pointer + &piProcInfo); // receives PROCESS_INFORMATION + + if (!processStarted) { + throw Error("BlackBoxExec", "Unable to start program `" + program + "'"); + } + + CloseHandle(piProcInfo.hThread); + // Stop ReadFile from blocking + CloseHandle(g_hChildStd_OUT_Wr); + // Just close the child's in pipe here + CloseHandle(g_hChildStd_IN_Rd); + + pipe_send = g_hChildStd_IN_Wr; + pipe_receive = g_hChildStd_OUT_Rd; +#else + const int READ = 0; + const int WRITE = 1; + int child_in[2]; + int child_out[2]; + pipe(child_in); + pipe(child_out); + + if (fork() != 0) { + close(child_in[READ]); + close(child_out[WRITE]); + + pipe_send = child_in[WRITE]; + int pipe_receive = child_out[READ]; + file_receive = fdopen(pipe_receive, "r"); + return; + } + close(STDIN_FILENO); + close(STDOUT_FILENO); + dup2(child_in[READ], STDIN_FILENO); + dup2(child_out[WRITE], STDOUT_FILENO); + close(child_in[WRITE]); + close(child_out[READ]); + + // Launch the program directly (no shell), passing the annotation arguments as + // its command-line arguments. + std::vector argv; + argv.push_back(const_cast(program.c_str())); + for (const std::string &a : args) { + argv.push_back(const_cast(a.c_str())); + } + argv.push_back(nullptr); + execvp(program.c_str(), argv.data()); + // execvp only returns on failure. + std::exit(127); +#endif +}; + +BlackBoxExec::~BlackBoxExec() { +#ifdef _WIN32 + CloseHandle(pipe_send); + CloseHandle(pipe_receive); +#else + close(pipe_send); + fclose(file_receive); +#endif +} + +void BlackBoxExec::run(const std::vector &int_in, + const std::vector &float_in, + std::vector &int_out, + std::vector &float_out) { + // Construct program input: comma-separated integers, a semicolon, then + // comma-separated floats, terminated by a newline (e.g. "5,-7;2.5,1.125\n"). + std::stringstream out; + out.precision(std::numeric_limits::max_digits10); + for (size_t i = 0; i < int_in.size(); ++i) { + if (i != 0) { + out << ","; + } + out << int_in[i]; + } + out << ";"; + for (size_t i = 0; i < float_in.size(); ++i) { + if (i != 0) { + out << ","; + } + out << float_in[i]; + } + out << "\n"; + std::string out_buf = out.str(); +#ifdef _WIN32 + // Write to process input pipe + BOOL success = + WriteFile(pipe_send, out_buf.c_str(), out_buf.size(), nullptr, nullptr); + assert(success); + + // Read output from process by pipe + char c[2] = {0, 0}; + std::ostringstream oss; + while (c[0] != '\n') { + DWORD count = 0; + BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); + if (!success) { + throw Error( + "BlackBoxExec", + "Reading blackbox process output from pipe resulted did not succeed"); + } else if (count == 0) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + assert(count == 1); + oss << c[0]; + } + std::string in_buffer(oss.str()); +#else + // Write to process input pipe + ssize_t bytes_written = write(pipe_send, out_buf.c_str(), out_buf.size()); + if (bytes_written != static_cast(out_buf.size())) { + throw Error("BlackBoxExec", + "Failed to write the full request to the blackbox process."); + } + + // Read from process output pipe + char *str = NULL; + size_t size = 0; + + if (getline(&str, &size, file_receive) == -1) { + throw Error( + "BlackBoxExec", + "Reading blackbox process output from pipe resulted in error no. " + + std::to_string(errno)); + } + std::string in_buffer(str); + free(str); +#endif + // Parse the response in a single left-to-right pass: comma-separated + // integers, a semicolon, then comma-separated floats (e.g. "5,-7;2.5,1.125\n"). + const char *p = in_buffer.c_str(); + char *end = nullptr; + auto skip_ws = [](const char *&q) { + while (*q == ' ' || *q == '\t' || *q == '\r') { + ++q; + } + }; + for (size_t i = 0; i < int_out.size(); ++i) { + long long v = std::strtoll(p, &end, 10); + if (end == p) { + throw Error("BlackBoxExec", "Failed to read output integer " + + std::to_string(i) + + " from blackbox process output, " + + std::to_string(int_out.size()) + + " integer values where expected."); + } + int_out[i] = static_cast(v); + p = end; + skip_ws(p); + if (*p == ',') { + ++p; + } + } + skip_ws(p); + if (*p != ';') { + throw Error("BlackBoxExec", + "Blackbox process response is missing the `;' separator between " + "the integer and floating point outputs."); + } + ++p; + for (size_t i = 0; i < float_out.size(); ++i) { + double v = std::strtod(p, &end); + if (end == p) { + throw Error("BlackBoxExec", "Failed to read output float " + + std::to_string(i) + + " from blackbox process output, " + + std::to_string(float_out.size()) + + " floating point values where expected."); + } + float_out[i] = v; + p = end; + skip_ws(p); + if (*p == ',') { + ++p; + } + } +} + +ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { + if (int_input.assigned() +#ifdef GECODE_HAS_FLOAT_VARS + && float_input.assigned() +#endif + ) { + std::vector int_in(int_input.size()); + std::vector int_out(int_output.size()); + // std::cerr << "Black Box Fn input: "; + for (int i = 0; i < int_in.size(); i++) { + // std::cerr << int_input[i].val() << " "; + int_in[i] = int_input[i].val(); + } + std::vector float_in; + std::vector float_out; +#ifdef GECODE_HAS_FLOAT_VARS + float_in.resize(float_input.size()); + float_out.resize(float_output.size()); + for (int i = 0; i < float_in.size(); i++) { + // std::cerr << float_input[i].val() << " "; + float_in[i] = float_input[i].val().med(); + } +#endif + // std::cerr << std::endl; + + black_box()->run(int_in, float_in, int_out, float_out); + + // std::cerr << "Black Box Fn output: "; + for (int i = 0; i < int_out.size(); i++) { + // std::cerr << int_out[i] << " "; + GECODE_ME_CHECK(int_output[i].eq(home, static_cast(int_out[i]))); + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < float_out.size(); i++) { + // std::cerr << float_out[i] << " "; + GECODE_ME_CHECK(float_output[i].eq(home, float_out[i])); + } +#endif + // std::cerr << std::endl; + + return home.ES_SUBSUMED(*this); + } + return ES_FIX; +} + +ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { + std::vector int_in(ivar.size() * 2); + std::vector int_out(ivar.size() * 2); + // std::cerr << "Black Box Bounds Fn input: "; + for (int i = 0; i < ivar.size(); i++) { + // std::cerr << ivar[i].min() << " " << ivar[i].max() << " "; + int_in[i*2] = ivar[i].min(); + int_in[i*2+1] = ivar[i].max(); + } + std::vector float_in; + std::vector float_out; +#ifdef GECODE_HAS_FLOAT_VARS + float_in.resize(fvar.size() * 2); + float_out.resize(fvar.size() * 2); + for (int i = 0; i < fvar.size(); i++) { + // std::cerr << fvar[i].min() << " " << fvar[i].max() << " "; + float_in[i*2] = fvar[i].min(); + float_in[i*2+1] = fvar[i].max(); + } +#endif + // std::cerr << std::endl; + + black_box()->run(int_in, float_in, int_out, float_out); + + // std::cerr << "Black Box Fn output: "; + for (int i = 0; i < ivar.size(); i++) { + // std::cerr << int_out[i*2] << ".." << int_out[i*2+1] << " "; + GECODE_ME_CHECK(ivar[i].gq(home, static_cast(int_out[i*2]))); + GECODE_ME_CHECK(ivar[i].lq(home, static_cast(int_out[i*2+1]))); + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) { + // std::cerr << float_out[i*2] << ".." << float_out[i*2+1] << " "; + GECODE_ME_CHECK(fvar[i].gq(home, float_out[i*2])); + GECODE_ME_CHECK(fvar[i].lq(home, float_out[i*2+1])); + } +#endif + // std::cerr << std::endl; + + return ES_NOFIX; +} + +void blackbox(Home home, const IntVarArgs &int_in, const IntVarArgs &int_out, +#ifdef GECODE_HAS_FLOAT_VARS + const FloatVarArgs &float_in, const FloatVarArgs &float_out, +#endif + const std::string &mode, const std::string &instantiation, + const std::vector &args) { + ViewArray int_input(home, int_in); + ViewArray int_output(home, int_out); +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray float_input(home, float_in); + ViewArray float_output(home, float_out); +#endif + + if (home.failed()) + return; + PostInfo pi(home); + ExecStatus es = BlackBox::post(home, int_input, int_output, +#ifdef GECODE_HAS_FLOAT_VARS + float_input, float_output, +#endif + mode, instantiation, args); + GECODE_ES_FAIL(es); +} + +/// Parse the flat reason and mark, per channel, the variables whose bounds the +/// propagator depends on (the variables that appear as literals in any reason). +/// \a sub_int / \a sub_float are filled with one boolean per variable. Variable +/// indices in the reason are 1-based over the combined variable list, integer +/// variables first, then float variables. +/// +/// The flat reason is a concatenation of one entry per variable, each entry +/// being `[idx, |R_lb|, (var, bnd)..., |R_ub|, (var, bnd)...]`. An empty reason +/// falls back to subscribing to every variable. +static void reason_subscriptions(const std::vector &reason, int n_int, + int n_float, SharedArray &sub_int, + SharedArray &sub_float) { + const bool all = reason.empty(); + for (int i = 0; i < n_int; i++) { + sub_int[i] = all; + } + for (int i = 0; i < n_float; i++) { + sub_float[i] = all; + } + if (all) { + return; + } + + size_t pos = 0; + while (pos < reason.size()) { + pos++; // idx: the variable being explained (not needed for subscription) + for (int side = 0; side < 2; side++) { // lower- then upper-bound literals + int count = reason[pos++]; + for (int k = 0; k < count; k++) { + int var = reason[pos++]; // 1-based combined variable index + pos++; // bound code (per-variable granularity only) + if (var >= 1 && var <= n_int) { + sub_int[var - 1] = true; + } else if (var > n_int && var <= n_int + n_float) { + sub_float[var - 1 - n_int] = true; + } + } + } + } +} + +void blackbox_bounds(Home home, const IntVarArgs &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + const FloatVarArgs &fvar, +#endif + const std::string &mode, const std::string &instantiation, + const std::vector &args, + const std::vector &reason) { + ViewArray int_var(home, ivar); +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray float_var(home, fvar); + int n_float = fvar.size(); +#else + int n_float = 0; +#endif + + // Determine which variables the propagator depends on, so it is only + // subscribed (and thus scheduled) on the bounds mentioned in the reason. The + // marking is constant and shared between all copies of the propagator. + SharedArray sub_int(ivar.size()); + SharedArray sub_float(n_float); + reason_subscriptions(reason, ivar.size(), n_float, sub_int, sub_float); + + if (home.failed()) + return; + PostInfo pi(home); + ExecStatus es = BlackBoxBounds::post(home, int_var, +#ifdef GECODE_HAS_FLOAT_VARS + float_var, +#endif + sub_int, +#ifdef GECODE_HAS_FLOAT_VARS + sub_float, +#endif + mode, instantiation, args); + GECODE_ES_FAIL(es); +} + +} // namespace FlatZinc +} // namespace Gecode diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh new file mode 100644 index 0000000000..5e7bf5edd7 --- /dev/null +++ b/gecode/flatzinc/blackbox.hh @@ -0,0 +1,400 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Copyright: + * Jip J. Dekker, 2026 + * + * This file is part of Gecode, the generic constraint + * development environment: + * http://www.gecode.org + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __FLATZINC_BLACKBOX_HH__ +#define __FLATZINC_BLACKBOX_HH__ + +#include +#include +#include +#include + +#include +#include +#ifdef GECODE_HAS_FLOAT_VARS +#include +#endif + +#ifdef _WIN32 +#define NOMINMAX // Ensure the words min/max remain available +#include +#else +// NOLINTNEXTLINE(bugprone-reserved-identifier) +#define __stdcall +#endif + +namespace Gecode { +namespace FlatZinc { + +/// Abstract class implemented by different methods to run blackbox functions +class BlackBoxFn : public SharedHandle::Object { +public: + virtual void run(const std::vector &int_in, + const std::vector &float_in, + std::vector &int_out, + std::vector &float_out) = 0; +}; + +/// Implementation of a black box function that dynamically loads a library and +/// run a contained function. +class BlackBoxDLL : public BlackBoxFn { +public: + BlackBoxDLL(const std::string &name, const std::vector &args); + ~BlackBoxDLL(); + void run(const std::vector &int_in, + const std::vector &float_in, std::vector &int_out, + std::vector &float_out) override { + dll_fzn_blackbox(int_in.data(), int_in.size(), float_in.data(), + float_in.size(), int_out.data(), int_out.size(), + float_out.data(), float_out.size()); + } + +protected: + void *library; + void(__stdcall *dll_fzn_blackbox)(const int64_t *, size_t, const double *, + size_t, int64_t *, size_t, double *, size_t); +}; + +/// Implementation of a black function that starts a seperate process to +/// repeatedly run a blackbox function, communication I/O over pipe. +class BlackBoxExec : public BlackBoxFn { +public: + BlackBoxExec(const std::string &program, const std::vector &args); + ~BlackBoxExec(); + void run(const std::vector &int_in, + const std::vector &float_in, std::vector &int_out, + std::vector &float_out) override; + +protected: +#ifdef _WIN32 + HANDLE pipe_send; + HANDLE pipe_receive; +#else + int pipe_send; + FILE *file_receive; +#endif +}; + +class BlackBoxHandle : public SharedHandle { +public: + BlackBoxHandle(BlackBoxFn *fn) : SharedHandle() { object(fn); } + BlackBoxHandle(const BlackBoxHandle &handle) : SharedHandle(handle) {} + BlackBoxHandle &operator=(const BlackBoxHandle &handle) { + return static_cast(SharedHandle::operator=(handle)); + } + BlackBoxFn *operator()() { return static_cast(object()); }; +}; + +class BlackBox : public Propagator { +protected: + /// Integer variables considered as the integer input to the blackbox function + ViewArray int_input; + /// Integer variables set to the integer output of the blackbox function + ViewArray int_output; + +#ifdef GECODE_HAS_FLOAT_VARS + /// Floating-point variables considered as the integer input to the blackbox function + ViewArray float_input; + /// Floating-point variables set to the integer output of the blackbox function + ViewArray float_output; +#endif + + /// Handle to the implementation of the blackbox function + /// + /// The handle ensures that the function implementation can be shared between + /// copies of the propagator. + BlackBoxHandle black_box; + + /// Constructor for cloning \a p + BlackBox(Space &home, BlackBox &p) + : Propagator(home, p), black_box(p.black_box) { + int_input.update(home, p.int_input); + int_output.update(home, p.int_output); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.update(home, p.float_input); + float_output.update(home, p.float_output); +#endif + } + +public: + /// Constructor for creation + BlackBox(Home home, ViewArray &int_in, + ViewArray &int_out, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray &float_in, + ViewArray &float_out, +#endif + BlackBoxFn *black_box) + : Propagator(home), int_input(int_in), int_output(int_out), +#ifdef GECODE_HAS_FLOAT_VARS + float_input(float_in), float_output(float_out), +#endif + black_box(black_box) { + int_input.subscribe(home, *this, Int::PC_INT_VAL); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.subscribe(home, *this, Float::PC_FLOAT_VAL); +#endif + } + /// Cost function (defined as exponential) + PropCost cost(const Space &home, const ModEventDelta &med) const override { + return PropCost::crazy(PropCost::HI, int_input.size() +#ifdef GECODE_HAS_FLOAT_VARS + + float_input.size() +#endif + ); + }; + /// Schedule function + void reschedule(Space &home) override { + int_input.cancel(home, *this, Int::PC_INT_VAL); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.cancel(home, *this, Float::PC_FLOAT_VAL); +#endif + } + /// Delete propagator and return its size + size_t dispose(Space &home) override { + int_input.cancel(home, *this, Int::PC_INT_VAL); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.cancel(home, *this, Float::PC_FLOAT_VAL); +#endif + (void)Propagator::dispose(home); + // destroy plugin container + return sizeof(*this); + }; + + ExecStatus propagate(Space &home, const ModEventDelta &) override; + + Propagator *copy(Space &home) override { + return new (home) BlackBox(home, *this); + } + + static ExecStatus post(Home home, ViewArray &int_input, + ViewArray &int_output, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray &float_input, + ViewArray &float_output, +#endif + const std::string &mode, + const std::string &instantiation, + const std::vector &args) { + BlackBoxFn *black_box(nullptr); + if (mode == "dll") { + black_box = new BlackBoxDLL(instantiation, args); + } else if (mode == "exec") { + black_box = new BlackBoxExec(instantiation, args); + } else { + throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); + } + + new (home) BlackBox(home, int_input, int_output, +#ifdef GECODE_HAS_FLOAT_VARS + float_input, float_output, +#endif + black_box); + return ES_OK; + } +}; + +class BlackBoxBounds : public Propagator { +protected: + /// Integer variables whose bounds are input and computed by the blackbox function (in order). + ViewArray ivar; + +#ifdef GECODE_HAS_FLOAT_VARS + /// Floating-point variables whose bounds are input and computed by the blackbox function (in order). + ViewArray fvar; +#endif + + /// For each variable in \a ivar, whether the propagator depends on its bounds + /// (derived from the reason). Only marked variables are subscribed, so the + /// propagator is scheduled precisely when one of the relevant bounds changes. + /// + /// The marking is constant during search and is shared between all copies of + /// the propagator. + SharedArray sub_int; +#ifdef GECODE_HAS_FLOAT_VARS + /// For each variable in \a fvar, whether the propagator depends on its bounds. + SharedArray sub_float; +#endif + + /// Handle to the implementation of the blackbox function + /// + /// The handle ensures that the function implementation can be shared between + /// copies of the propagator. + BlackBoxHandle black_box; + + /// Constructor for cloning \a p + BlackBoxBounds(Space &home, BlackBoxBounds &p) + : Propagator(home, p), sub_int(p.sub_int), +#ifdef GECODE_HAS_FLOAT_VARS + sub_float(p.sub_float), +#endif + black_box(p.black_box) { + ivar.update(home, p.ivar); +#ifdef GECODE_HAS_FLOAT_VARS + fvar.update(home, p.fvar); +#endif + } + +public: + /// Constructor for creation + BlackBoxBounds(Home home, ViewArray &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray &fvar, +#endif + SharedArray sub_int0, +#ifdef GECODE_HAS_FLOAT_VARS + SharedArray sub_float0, +#endif + BlackBoxFn *black_box) + : Propagator(home), ivar(ivar), +#ifdef GECODE_HAS_FLOAT_VARS + fvar(fvar), +#endif + sub_int(sub_int0), +#ifdef GECODE_HAS_FLOAT_VARS + sub_float(sub_float0), +#endif + black_box(black_box) { + for (int i = 0; i < ivar.size(); i++) { + if (sub_int[i]) { + ivar[i].subscribe(home, *this, Int::PC_INT_BND); + } + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) { + if (sub_float[i]) { + fvar[i].subscribe(home, *this, Float::PC_FLOAT_BND); + } + } +#endif + } + /// Cost function (defined as exponential) + PropCost cost(const Space &home, const ModEventDelta &med) const override { + return PropCost::crazy(PropCost::HI, ivar.size() +#ifdef GECODE_HAS_FLOAT_VARS + + fvar.size() +#endif + ); + }; + /// Schedule function + void reschedule(Space &home) override { + for (int i = 0; i < ivar.size(); i++) { + if (sub_int[i]) { + ivar[i].reschedule(home, *this, Int::PC_INT_BND); + } + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) { + if (sub_float[i]) { + fvar[i].reschedule(home, *this, Float::PC_FLOAT_BND); + } + } +#endif + } + /// Delete propagator and return its size + size_t dispose(Space &home) override { + for (int i = 0; i < ivar.size(); i++) { + if (sub_int[i]) { + ivar[i].cancel(home, *this, Int::PC_INT_BND); + } + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) { + if (sub_float[i]) { + fvar[i].cancel(home, *this, Float::PC_FLOAT_BND); + } + } +#endif + (void)Propagator::dispose(home); + // destroy plugin container + return sizeof(*this); + }; + + ExecStatus propagate(Space &home, const ModEventDelta &) override; + + Propagator *copy(Space &home) override { + return new (home) BlackBoxBounds(home, *this); + } + + static ExecStatus post(Home home, ViewArray &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray &fvar, +#endif + SharedArray sub_int, +#ifdef GECODE_HAS_FLOAT_VARS + SharedArray sub_float, +#endif + const std::string &mode, + const std::string &instantiation, + const std::vector &args) { + BlackBoxFn *black_box(nullptr); + if (mode == "dll") { + black_box = new BlackBoxDLL(instantiation, args); + } else if (mode == "exec") { + black_box = new BlackBoxExec(instantiation, args); + } else { + throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); + } + + new (home) BlackBoxBounds(home, ivar, +#ifdef GECODE_HAS_FLOAT_VARS + fvar, +#endif + sub_int, +#ifdef GECODE_HAS_FLOAT_VARS + sub_float, +#endif + black_box); + return ES_OK; + } +}; + +void blackbox(Home home, const IntVarArgs &int_in, const IntVarArgs &int_out, +#ifdef GECODE_HAS_FLOAT_VARS + const FloatVarArgs &float_in, const FloatVarArgs &float_out, +#endif + const std::string &mode, const std::string &instantiation, + const std::vector &args); + +void blackbox_bounds(Home home, const IntVarArgs &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + const FloatVarArgs &fvar, +#endif + const std::string &mode, const std::string &instantiation, + const std::vector &args, + const std::vector &reason); + +} // namespace FlatZinc +} // namespace Gecode + +#endif //__FLATZINC_BLACKBOX_HH__ diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn new file mode 100644 index 0000000000..c8d39f1c03 --- /dev/null +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn @@ -0,0 +1,13 @@ +predicate fzn_blackbox( + array[int] of var int: int_input, + array[int] of var float: float_input, + array[int] of var int: int_output, + array[int] of var float: float_output +) = gecode_blackbox(int_input, float_input, int_output, float_output); + +predicate gecode_blackbox( + array[int] of var int: int_input, + array[int] of var float: float_input, + array[int] of var int: int_output, + array[int] of var float: float_output +); diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn new file mode 100644 index 0000000000..af24151a7c --- /dev/null +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn @@ -0,0 +1,11 @@ +predicate fzn_blackbox_bounds( + array[int] of var int: int_input, + array[int] of var float: float_input, + array[int] of int: flat_reason, +) = gecode_blackbox_bounds(int_input, float_input, flat_reason); + +predicate gecode_blackbox_bounds( + array[int] of var int: int_input, + array[int] of var float: float_input, + array[int] of int: flat_reason, +); diff --git a/gecode/flatzinc/registry.cpp b/gecode/flatzinc/registry.cpp index 028427ab1e..8655fae1e3 100755 --- a/gecode/flatzinc/registry.cpp +++ b/gecode/flatzinc/registry.cpp @@ -47,6 +47,7 @@ #include #endif #include +#include namespace Gecode { namespace FlatZinc { @@ -1660,6 +1661,88 @@ namespace Gecode { namespace FlatZinc { member(s,x,y,s.arg2BoolVar(ce[2]),s.ann2ipl(ann)); } + /// Read a `blackbox_exec` / `blackbox_dll` source annotation into \a mode, + /// \a instantiation (the executable/library) and \a args (its argument + /// list). Supports both the single-argument form (no arguments) and the + /// `(target, args)` form. + void blackbox_source(AST::Node* ann, std::string& mode, + std::string& instantiation, + std::vector& args) { + AST::Call* c = nullptr; + if (ann->hasCall("blackbox_dll")) { + c = ann->getCall("blackbox_dll"); + mode = "dll"; + } else if (ann->hasCall("blackbox_exec")) { + c = ann->getCall("blackbox_exec"); + mode = "exec"; + } else { + throw FlatZinc::Error("Registry", + "Blackbox constraint is missing a valid annotation specifying execution method."); + } + // For a single-argument call `args` is the bare argument node; for the + // `(target, args)` form it is an array of the two arguments. + if (AST::Array* arr = dynamic_cast(c->args)) { + instantiation = arr->a[0]->getString(); + if (arr->a.size() > 1) { + AST::Array* al = arr->a[1]->getArray(); + for (unsigned int i = 0; i < al->a.size(); i++) { + args.push_back(al->a[i]->getString()); + } + } + } else { + instantiation = c->args->getString(); + } + } + + void p_blackbox(FlatZincSpace& s, const ConExpr& ce, AST::Node* ann) { + std::string mode; + std::string instantiation; + std::vector args; + blackbox_source(ann, mode, instantiation, args); + IntVarArgs int_input = s.arg2intvarargs(ce[0]); + IntVarArgs int_output = s.arg2intvarargs(ce[2]); +#ifdef GECODE_HAS_FLOAT_VARS + FloatVarArgs float_input = s.arg2floatvarargs(ce[1]); + FloatVarArgs float_output = s.arg2floatvarargs(ce[3]); +#else + if (!ce[1]->getArray()->a.empty() || !ce[3]->getArray()->a.empty()) { + throw FlatZinc::Error("Registry", + "Blackbox propagator cannot use floating point values when Gecode is compiled without floating point decision variable support."); + } +#endif + FlatZinc::blackbox(s, int_input, int_output, +#ifdef GECODE_HAS_FLOAT_VARS +float_input, float_output, +#endif + mode, instantiation, args); + } + + void p_blackbox_bounds(FlatZincSpace& s, const ConExpr& ce, AST::Node* ann) { + std::string mode; + std::string instantiation; + std::vector args; + blackbox_source(ann, mode, instantiation, args); + IntVarArgs ivar = s.arg2intvarargs(ce[0]); +#ifdef GECODE_HAS_FLOAT_VARS + FloatVarArgs fvar = s.arg2floatvarargs(ce[1]); +#else + if (!ce[1]->getArray()->a.empty()) { + throw FlatZinc::Error("Registry", + "Blackbox propagator cannot use floating point values when Gecode is compiled without floating point decision variable support."); + } +#endif + IntArgs flat_reason = s.arg2intargs(ce[2]); + std::vector reason(flat_reason.size()); + for (int i = 0; i < flat_reason.size(); i++) { + reason[i] = flat_reason[i]; + } + FlatZinc::blackbox_bounds(s, ivar, +#ifdef GECODE_HAS_FLOAT_VARS +fvar, +#endif + mode, instantiation, args, reason); + } + class IntPoster { public: IntPoster(void) { @@ -1848,6 +1931,9 @@ namespace Gecode { namespace FlatZinc { registry().add("gecode_member_int_reif",&p_member_int_reif); registry().add("member_bool",&p_member_bool); registry().add("gecode_member_bool_reif",&p_member_bool_reif); + + registry().add("gecode_blackbox", &p_blackbox); + registry().add("gecode_blackbox_bounds", &p_blackbox_bounds); } }; IntPoster __int_poster; From a9ebff8a644d94e5c56acbe9a0ee7d8438e9b850 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Fri, 10 Jul 2026 20:53:02 +0200 Subject: [PATCH 02/14] Harden FlatZinc blackbox runtime Strengthen blackbox propagation, ownership, disposal, and error handling around external backends. --- gecode/flatzinc/blackbox.cpp | 991 +++++++++++++++++++++++++++++------ gecode/flatzinc/blackbox.hh | 87 ++- 2 files changed, 897 insertions(+), 181 deletions(-) diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp index 7c662cd451..e9419ad3d4 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox.cpp @@ -37,61 +37,267 @@ #include #include +#include +#include #include +#include #include #include #include #include -#ifndef _WIN32 -#include +#ifdef _WIN32 +#define NOMINMAX // Ensure the words min/max remain available +#include +#else #include -#include +#include +#include +#include +#include +#include +#include +#include #include +extern char **environ; +#endif + +#ifdef GECODE_HAS_THREADS +#include #endif namespace Gecode { namespace FlatZinc { +namespace { + +#ifdef _WIN32 +std::wstring +utf8_to_wide(const std::string &s) { + if (s.empty()) { + return std::wstring(); + } + int n = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.c_str(), + static_cast(s.size()), NULL, 0); + if (n == 0) { + throw Error("Blackbox", "Invalid UTF-8 string in blackbox path or argument"); + } + std::wstring w(static_cast(n), L'\0'); + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.c_str(), + static_cast(s.size()), &w[0], n) == 0) { + throw Error("Blackbox", "Invalid UTF-8 string in blackbox path or argument"); + } + return w; +} + +std::string +windows_error(const std::string &prefix, DWORD err) { + return prefix + " (Windows error " + std::to_string(err) + ")"; +} + +void +close_library(void *library) { + if (library != nullptr) { + FreeLibrary(static_cast(library)); + } +} +#else +void +close_library(void *library) { + if (library != nullptr) { + dlclose(library); + } +} + +int +set_cloexec(int fd) { + int flags = fcntl(fd, F_GETFD); + if (flags == -1) { + return -1; + } + return fcntl(fd, F_SETFD, flags | FD_CLOEXEC); +} + +int +dup_cloexec(int fd, int min_fd) { + int nfd; +#ifdef F_DUPFD_CLOEXEC + nfd = fcntl(fd, F_DUPFD_CLOEXEC, min_fd); + if (nfd != -1) { + return nfd; + } + if (errno != EINVAL) { + return -1; + } +#endif + nfd = fcntl(fd, F_DUPFD, min_fd); + if (nfd == -1) { + return -1; + } + if (set_cloexec(nfd) != 0) { + int e = errno; + ::close(nfd); + errno = e; + return -1; + } + return nfd; +} + +int +move_from_standard_fd(int fd) { + if (fd > STDERR_FILENO) { + return fd; + } + int nfd = dup_cloexec(fd, STDERR_FILENO + 1); + if (nfd == -1) { + return -1; + } + ::close(fd); + return nfd; +} + +int +create_socketpair(int sv[2]) { +#ifdef SOCK_CLOEXEC + if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == 0) { + return 0; + } + if (errno != EINVAL) { + return -1; + } +#endif + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) { + return -1; + } + if ((set_cloexec(sv[0]) != 0) || (set_cloexec(sv[1]) != 0)) { + int e = errno; + ::close(sv[0]); + ::close(sv[1]); + errno = e; + return -1; + } + return 0; +} + +ssize_t +send_no_sigpipe(int fd, const char *data, size_t size) { +#ifdef MSG_NOSIGNAL + return send(fd, data, size, MSG_NOSIGNAL); +#else +#ifdef SO_NOSIGPIPE + return send(fd, data, size, 0); +#else + sigset_t block; + sigset_t old; + sigset_t pending; + sigemptyset(&block); + sigaddset(&block, SIGPIPE); + bool blocked = false; + bool was_pending = false; + if (pthread_sigmask(SIG_BLOCK, &block, &old) == 0) { + blocked = true; + if (sigpending(&pending) == 0) { + was_pending = sigismember(&pending, SIGPIPE) == 1; + } + } + ssize_t n = send(fd, data, size, 0); + if ((n == -1) && (errno == EPIPE) && !was_pending) { + const struct timespec timeout = {0, 0}; + sigtimedwait(&block, NULL, &timeout); + } + if (blocked) { + pthread_sigmask(SIG_SETMASK, &old, NULL); + } + return n; +#endif +#endif +} +#endif + +int +checked_int(long long v, const char *source, size_t i) { + if ((v < Int::Limits::min) || (v > Int::Limits::max) || + (v < std::numeric_limits::min()) || + (v > std::numeric_limits::max())) { + throw Error("Blackbox", std::string(source) + " integer " + + std::to_string(i) + + " is outside Gecode's integer range"); + } + return static_cast(v); +} + +} // namespace + BlackBoxDLL::BlackBoxDLL(const std::string &name, - const std::vector &args) { + const std::vector &args) + : library(nullptr), dll_fzn_blackbox(nullptr) { std::string loadError; + void *loaded = nullptr; #ifdef _WIN32 - library = LoadLibraryA(name.c_str()); - if (!library) { + std::wstring wname = utf8_to_wide(name); + loaded = LoadLibraryW(wname.c_str()); + if (!loaded) { + DWORD err = GetLastError(); loadError = std::string("unable to locate library `") + name + "'"; - library = LoadLibraryA((std::string(name) + ".dll").c_str()); + std::wstring wdll = utf8_to_wide(name + ".dll"); + loaded = LoadLibraryW(wdll.c_str()); + if (!loaded) { + err = GetLastError(); + } } - if (!library) { - library = LoadLibraryA((std::string("lib") + name + ".dll").c_str()); + if (!loaded) { + std::wstring wlibdll = utf8_to_wide(std::string("lib") + name + ".dll"); + loaded = LoadLibraryW(wlibdll.c_str()); + if (!loaded) { + loadError += " (" + windows_error("LoadLibraryW failed", err) + ")"; + } } #else - library = dlopen(name.c_str(), RTLD_LAZY); - if (!library) { + loaded = dlopen(name.c_str(), RTLD_LAZY); + if (!loaded) { loadError = std::string(dlerror()); - library = dlopen((name + ".so").c_str(), RTLD_NOW); + loaded = dlopen((name + ".so").c_str(), RTLD_NOW); + } + if (!loaded) { + loaded = dlopen((std::string("lib") + name + ".so").c_str(), RTLD_NOW); } - if (!library) { - library = dlopen((std::string("lib") + name + ".so").c_str(), RTLD_NOW); +#ifdef __APPLE__ + if (!loaded) { + loaded = dlopen((name + ".dylib").c_str(), RTLD_NOW); } + if (!loaded) { + loaded = dlopen((std::string("lib") + name + ".dylib").c_str(), RTLD_NOW); + } +#endif #endif - if (!library) { + if (!loaded) { throw Error("Blackbox", "Unable to open dynamic library: " + loadError); } - // find symbol for blacbox function + // find symbol for blackbox function #ifdef _WIN32 dll_fzn_blackbox = reinterpret_cast( - GetProcAddress((HMODULE)library, "fzn_blackbox")); + GetProcAddress((HMODULE)loaded, "fzn_blackbox")); +#if defined(_M_IX86) || defined(__i386__) + if (!dll_fzn_blackbox) { + dll_fzn_blackbox = reinterpret_cast( + GetProcAddress((HMODULE)loaded, "_fzn_blackbox@32")); + } + if (!dll_fzn_blackbox) { + dll_fzn_blackbox = reinterpret_cast( + GetProcAddress((HMODULE)loaded, "fzn_blackbox@32")); + } +#endif std::string symError = "."; #else - *(void **)(&dll_fzn_blackbox) = dlsym(library, "fzn_blackbox"); + *(void **)(&dll_fzn_blackbox) = dlsym(loaded, "fzn_blackbox"); std::string symError(": "); if (!dll_fzn_blackbox) { symError += std::string(dlerror()); } #endif if (!dll_fzn_blackbox) { + close_library(loaded); throw Error("Blackbox", "Unable to find symbol `fzn_blackbox` in dynamic library" + symError); @@ -99,36 +305,167 @@ BlackBoxDLL::BlackBoxDLL(const std::string &name, // Optionally call the initialisation function with the given arguments. It is // not an error for the library to omit `fzn_initialize`. - void(__stdcall *dll_fzn_initialize)(const char **, size_t) = nullptr; + void(GECODE_BLACKBOX_CALL *dll_fzn_initialize)(const char **, size_t) = + nullptr; #ifdef _WIN32 dll_fzn_initialize = reinterpret_cast( - GetProcAddress((HMODULE)library, "fzn_initialize")); + GetProcAddress((HMODULE)loaded, "fzn_initialize")); +#if defined(_M_IX86) || defined(__i386__) + if (!dll_fzn_initialize) { + dll_fzn_initialize = reinterpret_cast( + GetProcAddress((HMODULE)loaded, "_fzn_initialize@8")); + } + if (!dll_fzn_initialize) { + dll_fzn_initialize = reinterpret_cast( + GetProcAddress((HMODULE)loaded, "fzn_initialize@8")); + } +#endif #else - *(void **)(&dll_fzn_initialize) = dlsym(library, "fzn_initialize"); + *(void **)(&dll_fzn_initialize) = dlsym(loaded, "fzn_initialize"); #endif - if (dll_fzn_initialize != nullptr) { - std::vector argv; - argv.reserve(args.size()); - for (const std::string &a : args) { - argv.push_back(a.c_str()); + try { + if (dll_fzn_initialize != nullptr) { + std::vector argv; + argv.reserve(args.size()); + for (const std::string &a : args) { + argv.push_back(a.c_str()); + } + dll_fzn_initialize(argv.data(), argv.size()); } - dll_fzn_initialize(argv.data(), argv.size()); + } catch (...) { + close_library(loaded); + throw; } + library = loaded; } BlackBoxDLL::~BlackBoxDLL() { - if (library) { + close_library(library); +} + +class BlackBoxExec::Session { +protected: #ifdef _WIN32 - FreeLibrary((HMODULE)library); + HANDLE job; + HANDLE process; + HANDLE pipe_send; + HANDLE pipe_receive; #else - dlclose(library); + pid_t child; + int pipe_send; + FILE *file_receive; +#endif +#ifdef GECODE_HAS_THREADS + std::thread::id owner; +#endif + + static std::string last_error(const std::string &prefix) { +#ifdef _WIN32 + return prefix + " (Windows error " + std::to_string(GetLastError()) + ")"; +#else + return prefix + " (errno " + std::to_string(errno) + ")"; #endif } -} -BlackBoxExec::BlackBoxExec(const std::string &program, - const std::vector &args) { #ifdef _WIN32 + static std::wstring quote_argument(const std::wstring &arg) { + std::wstring q(L"\""); + unsigned int backslashes = 0; + for (wchar_t ch : arg) { + if (ch == L'\\') { + backslashes++; + } else if (ch == L'"') { + q.append(backslashes * 2 + 1, L'\\'); + q += ch; + backslashes = 0; + } else { + q.append(backslashes, L'\\'); + q += ch; + backslashes = 0; + } + } + q.append(backslashes * 2, L'\\'); + q += L'"'; + return q; + } +#else + static bool reap_child(pid_t pid, int &status) { + pid_t r; + do { + r = waitpid(pid, &status, WNOHANG); + } while ((r == -1) && (errno == EINTR)); + return (r == pid) || ((r == -1) && (errno == ECHILD)); + } + + static bool wait_child(pid_t pid, int &status, int attempts) { + for (int i = 0; i < attempts; i++) { + if (reap_child(pid, status)) { + return true; + } + usleep(10000); + } + return reap_child(pid, status); + } + + static void terminate_child(pid_t pid) { + if (pid <= 0) { + return; + } + int status = 0; + if (wait_child(pid, status, 100)) { + return; + } + if (kill(-pid, SIGTERM) != 0) { + kill(pid, SIGTERM); + } + if (wait_child(pid, status, 100)) { + return; + } + if (kill(-pid, SIGKILL) != 0) { + kill(pid, SIGKILL); + } + do { + if (waitpid(pid, &status, 0) != -1) { + return; + } + } while (errno == EINTR); + } +#endif + +public: + Session(const std::string &program, const std::vector &args) +#ifdef GECODE_HAS_THREADS +#ifdef _WIN32 + : job(NULL), process(NULL), pipe_send(NULL), pipe_receive(NULL), + owner(std::this_thread::get_id()) +#else + : child(-1), pipe_send(-1), file_receive(NULL), + owner(std::this_thread::get_id()) +#endif +#else +#ifdef _WIN32 + : job(NULL), process(NULL), pipe_send(NULL), pipe_receive(NULL) +#else + : child(-1), pipe_send(-1), file_receive(NULL) +#endif +#endif + { +#ifdef _WIN32 + // Build the command line before opening OS handles so allocation/conversion + // failures cannot leak partially constructed process state. + std::wstring program_w = utf8_to_wide(program); + std::wstring prog = quote_argument(program_w); + for (const std::string &a : args) { + prog += L" "; + prog += quote_argument(utf8_to_wide(a)); + } + std::vector cmdline(prog.begin(), prog.end()); + cmdline.push_back(L'\0'); + + SIZE_T attr_size = 0; + InitializeProcThreadAttributeList(NULL, 1, 0, &attr_size); + std::vector attr_buf(attr_size); + SECURITY_ATTRIBUTES saAttr; saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); saAttr.bInheritHandle = TRUE; @@ -138,117 +475,462 @@ BlackBoxExec::BlackBoxExec(const std::string &program, HANDLE g_hChildStd_IN_Wr = NULL; HANDLE g_hChildStd_OUT_Rd = NULL; HANDLE g_hChildStd_OUT_Wr = NULL; + HANDLE g_hChildStd_ERR_Wr = NULL; // Create a pipe for the child process's STDOUT. - if (!CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0)) - std::cerr << "Stdout CreatePipe" << std::endl; + if (!CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0)) { + throw Error("BlackBoxExec", last_error("Stdout CreatePipe failed")); + } // Ensure the read handle to the pipe for STDOUT is not inherited. - if (!SetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0)) - std::cerr << "Stdout SetHandleInformation" << std::endl; + if (!SetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0)) { + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + throw Error("BlackBoxExec", + last_error("Stdout SetHandleInformation failed")); + } // Create a pipe for the child process's STDIN - if (!CreatePipe(&g_hChildStd_IN_Rd, &g_hChildStd_IN_Wr, &saAttr, 0)) - std::cerr << "Stdin CreatePipe" << std::endl; + if (!CreatePipe(&g_hChildStd_IN_Rd, &g_hChildStd_IN_Wr, &saAttr, 0)) { + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + throw Error("BlackBoxExec", last_error("Stdin CreatePipe failed")); + } // Ensure the write handle to the pipe for STDIN is not inherited. - if (!SetHandleInformation(g_hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0)) - std::cerr << "Stdin SetHandleInformation" << std::endl; + if (!SetHandleInformation(g_hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0)) { + DWORD err = GetLastError(); + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_IN_Wr); + throw Error("BlackBoxExec", windows_error( + "Stdin SetHandleInformation failed", err)); + } + + HANDLE parent_stderr = GetStdHandle(STD_ERROR_HANDLE); + if ((parent_stderr != NULL) && (parent_stderr != INVALID_HANDLE_VALUE)) { + if (!DuplicateHandle(GetCurrentProcess(), parent_stderr, + GetCurrentProcess(), &g_hChildStd_ERR_Wr, 0, TRUE, + DUPLICATE_SAME_ACCESS)) { + DWORD err = GetLastError(); + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_IN_Wr); + throw Error("BlackBoxExec", + windows_error("stderr DuplicateHandle failed", err)); + } + } PROCESS_INFORMATION piProcInfo; - STARTUPINFOA siStartInfo; + STARTUPINFOEXW siStartInfo; // Set up members of the PROCESS_INFORMATION structure. ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); // Set up members of the STARTUPINFO structure. // This structure specifies the STDIN and STDOUT handles for redirection. - ZeroMemory(&siStartInfo, sizeof(STARTUPINFOA)); - siStartInfo.cb = sizeof(STARTUPINFOA); - siStartInfo.hStdOutput = g_hChildStd_OUT_Wr; - siStartInfo.hStdInput = g_hChildStd_IN_Rd; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - // Build the command line: the program followed by the (quoted) arguments. - std::string prog = program; - for (const std::string &a : args) { - prog += " \""; - for (char ch : a) { - if (ch == '"' || ch == '\\') { - prog += '\\'; - } - prog += ch; - } - prog += '"'; + ZeroMemory(&siStartInfo, sizeof(STARTUPINFOEXW)); + siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEXW); + siStartInfo.StartupInfo.hStdOutput = g_hChildStd_OUT_Wr; + siStartInfo.StartupInfo.hStdInput = g_hChildStd_IN_Rd; + siStartInfo.StartupInfo.hStdError = g_hChildStd_ERR_Wr; + siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; + + HANDLE inherit_handles[3] = {g_hChildStd_IN_Rd, g_hChildStd_OUT_Wr, NULL}; + DWORD inherit_count = 2; + if (g_hChildStd_ERR_Wr != NULL) { + inherit_handles[inherit_count++] = siStartInfo.StartupInfo.hStdError; + } + + siStartInfo.lpAttributeList = + reinterpret_cast(attr_buf.data()); + if (!InitializeProcThreadAttributeList(siStartInfo.lpAttributeList, 1, 0, + &attr_size)) { + DWORD err = GetLastError(); + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_IN_Wr); + if (g_hChildStd_ERR_Wr != NULL) + CloseHandle(g_hChildStd_ERR_Wr); + throw Error("BlackBoxExec", + windows_error("InitializeProcThreadAttributeList failed", err)); } + if (!UpdateProcThreadAttribute(siStartInfo.lpAttributeList, 0, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + inherit_handles, + sizeof(HANDLE) * inherit_count, + NULL, NULL)) { + DWORD err = GetLastError(); + DeleteProcThreadAttributeList(siStartInfo.lpAttributeList); + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_IN_Wr); + if (g_hChildStd_ERR_Wr != NULL) + CloseHandle(g_hChildStd_ERR_Wr); + throw Error("BlackBoxExec", + windows_error("PROC_THREAD_ATTRIBUTE_HANDLE_LIST failed", err)); + } + BOOL processStarted = - CreateProcessA(nullptr, - prog.data(), // command line - nullptr, // process security attributes - nullptr, // primary thread security attributes - TRUE, // handles are inherited - 0, // creation flags - nullptr, // use parent's environment - nullptr, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &piProcInfo); // receives PROCESS_INFORMATION + CreateProcessW(nullptr, + cmdline.data(), // command line + nullptr, // process security attributes + nullptr, // primary thread security attributes + TRUE, // handles from attribute list + EXTENDED_STARTUPINFO_PRESENT | CREATE_SUSPENDED, + nullptr, // use parent's environment + nullptr, // use parent's current directory + &siStartInfo.StartupInfo, + &piProcInfo); // receives PROCESS_INFORMATION + DeleteProcThreadAttributeList(siStartInfo.lpAttributeList); if (!processStarted) { - throw Error("BlackBoxExec", "Unable to start program `" + program + "'"); + DWORD err = GetLastError(); + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_IN_Wr); + if (g_hChildStd_ERR_Wr != NULL) + CloseHandle(g_hChildStd_ERR_Wr); + throw Error("BlackBoxExec", windows_error("Unable to start program `" + + program + "'", err)); } + HANDLE process_job = CreateJobObjectW(NULL, NULL); + if (process_job != NULL) { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION job_info; + ZeroMemory(&job_info, sizeof(job_info)); + job_info.BasicLimitInformation.LimitFlags = + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (!SetInformationJobObject(process_job, JobObjectExtendedLimitInformation, + &job_info, sizeof(job_info)) || + !AssignProcessToJobObject(process_job, piProcInfo.hProcess)) { + CloseHandle(process_job); + process_job = NULL; + } + } + + if (ResumeThread(piProcInfo.hThread) == static_cast(-1)) { + DWORD err = GetLastError(); + if (process_job != NULL) { + TerminateJobObject(process_job, 1); + } else { + TerminateProcess(piProcInfo.hProcess, 1); + } + WaitForSingleObject(piProcInfo.hProcess, 5000); + CloseHandle(piProcInfo.hThread); + CloseHandle(piProcInfo.hProcess); + if (process_job != NULL) + CloseHandle(process_job); + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_IN_Wr); + if (g_hChildStd_ERR_Wr != NULL) + CloseHandle(g_hChildStd_ERR_Wr); + throw Error("BlackBoxExec", + windows_error("ResumeThread failed for blackbox process", err)); + } CloseHandle(piProcInfo.hThread); // Stop ReadFile from blocking CloseHandle(g_hChildStd_OUT_Wr); // Just close the child's in pipe here CloseHandle(g_hChildStd_IN_Rd); + if (g_hChildStd_ERR_Wr != NULL) + CloseHandle(g_hChildStd_ERR_Wr); pipe_send = g_hChildStd_IN_Wr; pipe_receive = g_hChildStd_OUT_Rd; + process = piProcInfo.hProcess; + job = process_job; #else const int READ = 0; const int WRITE = 1; - int child_in[2]; - int child_out[2]; - pipe(child_in); - pipe(child_out); - - if (fork() != 0) { - close(child_in[READ]); - close(child_out[WRITE]); - - pipe_send = child_in[WRITE]; - int pipe_receive = child_out[READ]; - file_receive = fdopen(pipe_receive, "r"); - return; + int child_in[2] = {-1, -1}; + int child_out[2] = {-1, -1}; + if (create_socketpair(child_in) != 0) { + throw Error("BlackBoxExec", last_error("stdin socket creation failed")); } - close(STDIN_FILENO); - close(STDOUT_FILENO); - dup2(child_in[READ], STDIN_FILENO); - dup2(child_out[WRITE], STDOUT_FILENO); - close(child_in[WRITE]); - close(child_out[READ]); + if (create_socketpair(child_out) != 0) { + ::close(child_in[READ]); + ::close(child_in[WRITE]); + throw Error("BlackBoxExec", last_error("stdout socket creation failed")); + } + int fds[4] = {child_in[READ], child_in[WRITE], + child_out[READ], child_out[WRITE]}; + for (int i = 0; i < 4; i++) { + int moved = move_from_standard_fd(fds[i]); + if (moved == -1) { + int e = errno; + for (int j = 0; j < 4; j++) + ::close(fds[j]); + errno = e; + throw Error("BlackBoxExec", + last_error("moving session descriptors away from stdio " + "failed")); + } + fds[i] = moved; + } + child_in[READ] = fds[0]; + child_in[WRITE] = fds[1]; + child_out[READ] = fds[2]; + child_out[WRITE] = fds[3]; - // Launch the program directly (no shell), passing the annotation arguments as - // its command-line arguments. std::vector argv; + argv.reserve(args.size() + 2); argv.push_back(const_cast(program.c_str())); for (const std::string &a : args) { argv.push_back(const_cast(a.c_str())); } argv.push_back(nullptr); - execvp(program.c_str(), argv.data()); - // execvp only returns on failure. - std::exit(127); + + posix_spawn_file_actions_t actions; + int err = posix_spawn_file_actions_init(&actions); + if (err != 0) { + ::close(child_in[READ]); + ::close(child_in[WRITE]); + ::close(child_out[READ]); + ::close(child_out[WRITE]); + errno = err; + throw Error("BlackBoxExec", last_error("spawn file action init failed")); + } + + posix_spawnattr_t attr; + err = posix_spawnattr_init(&attr); + if (err != 0) { + posix_spawn_file_actions_destroy(&actions); + ::close(child_in[READ]); + ::close(child_in[WRITE]); + ::close(child_out[READ]); + ::close(child_out[WRITE]); + errno = err; + throw Error("BlackBoxExec", last_error("spawn attribute init failed")); + } + + err = posix_spawnattr_setpgroup(&attr, 0); + if (err == 0) { + err = posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETPGROUP); + } + if (err == 0) { + err = posix_spawn_file_actions_adddup2(&actions, child_in[READ], + STDIN_FILENO); + } + if (err == 0) { + err = posix_spawn_file_actions_adddup2(&actions, child_out[WRITE], + STDOUT_FILENO); + } + if (err == 0) { + err = posix_spawn_file_actions_addclose(&actions, child_in[READ]); + } + if (err == 0) { + err = posix_spawn_file_actions_addclose(&actions, child_in[WRITE]); + } + if (err == 0) { + err = posix_spawn_file_actions_addclose(&actions, child_out[READ]); + } + if (err == 0) { + err = posix_spawn_file_actions_addclose(&actions, child_out[WRITE]); + } + if (err == 0) { + err = posix_spawnp(&child, program.c_str(), &actions, &attr, argv.data(), + environ); + } + posix_spawnattr_destroy(&attr); + posix_spawn_file_actions_destroy(&actions); + if (err != 0) { + ::close(child_in[READ]); + ::close(child_in[WRITE]); + ::close(child_out[READ]); + ::close(child_out[WRITE]); + child = -1; + errno = err; + throw Error("BlackBoxExec", last_error("starting blackbox process failed")); + } + + ::close(child_in[READ]); + ::close(child_out[WRITE]); + + pipe_send = child_in[WRITE]; +#ifdef SO_NOSIGPIPE + int nosigpipe = 1; + if (setsockopt(pipe_send, SOL_SOCKET, SO_NOSIGPIPE, &nosigpipe, + sizeof(nosigpipe)) != 0) { + ::close(pipe_send); + pipe_send = -1; + ::close(child_out[READ]); + terminate_child(child); + child = -1; + throw Error("BlackBoxExec", last_error("SO_NOSIGPIPE setup failed")); + } #endif -}; + file_receive = fdopen(child_out[READ], "r"); + if (file_receive == NULL) { + ::close(pipe_send); + pipe_send = -1; + ::close(child_out[READ]); + terminate_child(child); + child = -1; + throw Error("BlackBoxExec", last_error("fdopen failed")); + } + return; +#endif + } + + ~Session(void) { close(); } + + bool owned_by_current_thread(void) const { +#ifdef GECODE_HAS_THREADS + return owner == std::this_thread::get_id(); +#else + return true; +#endif + } -BlackBoxExec::~BlackBoxExec() { + std::string run(const std::string &out_buf) { #ifdef _WIN32 - CloseHandle(pipe_send); - CloseHandle(pipe_receive); + size_t written = 0; + while (written < out_buf.size()) { + DWORD count = 0; + DWORD remaining = + static_cast(out_buf.size() - written); + BOOL success = + WriteFile(pipe_send, out_buf.data() + written, remaining, &count, + nullptr); + if (!success || count == 0) { + throw Error("BlackBoxExec", + last_error("Writing blackbox process input failed")); + } + written += count; + } + + char c[2] = {0, 0}; + std::ostringstream oss; + while (c[0] != '\n') { + DWORD count = 0; + BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); + if (!success) { + throw Error( + "BlackBoxExec", + "Reading blackbox process output from pipe resulted did not succeed"); + } else if (count == 0) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + assert(count == 1); + oss << c[0]; + } + return oss.str(); #else - close(pipe_send); - fclose(file_receive); + const char *p = out_buf.c_str(); + size_t remaining = out_buf.size(); + while (remaining > 0) { + ssize_t n = send_no_sigpipe(pipe_send, p, remaining); + if (n < 0) { + if (errno == EINTR) { + continue; + } + throw Error("BlackBoxExec", + "Writing blackbox process input failed with errno " + + std::to_string(errno)); + } + if (n == 0) { + throw Error("BlackBoxExec", + "Writing blackbox process input wrote zero bytes"); + } + p += n; + remaining -= static_cast(n); + } + + char *str = NULL; + size_t size = 0; + errno = 0; + if (getline(&str, &size, file_receive) == -1) { + free(str); + if (feof(file_receive)) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + throw Error( + "BlackBoxExec", + "Reading blackbox process output from pipe resulted in error no. " + + std::to_string(errno)); + } + std::string in_buffer(str); + free(str); + return in_buffer; #endif + } + + void close(void) { +#ifdef _WIN32 + if (pipe_send != NULL) { + CloseHandle(pipe_send); + pipe_send = NULL; + } + if (pipe_receive != NULL) { + CloseHandle(pipe_receive); + pipe_receive = NULL; + } + if (process != NULL) { + DWORD wait = WaitForSingleObject(process, 1000); + if (wait == WAIT_TIMEOUT) { + if (job != NULL) { + TerminateJobObject(job, 1); + } else { + TerminateProcess(process, 1); + } + WaitForSingleObject(process, 5000); + } + CloseHandle(process); + process = NULL; + } + if (job != NULL) { + CloseHandle(job); + job = NULL; + } +#else + if (pipe_send != -1) { + ::close(pipe_send); + pipe_send = -1; + } + if (file_receive != NULL) { + fclose(file_receive); + file_receive = NULL; + } + if (child > 0) { + terminate_child(child); + child = -1; + } +#endif + } +}; + +BlackBoxExec::BlackBoxExec(const std::string &program0, + const std::vector &args0) + : program(program0), args(args0) {} + +BlackBoxExec::~BlackBoxExec(void) { + Support::Lock lock(mutex); + for (Session *s : sessions) { + delete s; + } + sessions.clear(); +} + +BlackBoxExec::Session &BlackBoxExec::session(void) { + Support::Lock lock(mutex); + for (Session *s : sessions) { + if (s->owned_by_current_thread()) { + return *s; + } + } + std::unique_ptr s(new Session(program, args)); + Session *r = s.get(); + sessions.push_back(r); + s.release(); + return *r; } void BlackBoxExec::run(const std::vector &int_in, @@ -274,51 +956,7 @@ void BlackBoxExec::run(const std::vector &int_in, } out << "\n"; std::string out_buf = out.str(); -#ifdef _WIN32 - // Write to process input pipe - BOOL success = - WriteFile(pipe_send, out_buf.c_str(), out_buf.size(), nullptr, nullptr); - assert(success); - - // Read output from process by pipe - char c[2] = {0, 0}; - std::ostringstream oss; - while (c[0] != '\n') { - DWORD count = 0; - BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); - if (!success) { - throw Error( - "BlackBoxExec", - "Reading blackbox process output from pipe resulted did not succeed"); - } else if (count == 0) { - throw Error("BlackBoxExec", - "Blackbox process provided an incomplete response"); - } - assert(count == 1); - oss << c[0]; - } - std::string in_buffer(oss.str()); -#else - // Write to process input pipe - ssize_t bytes_written = write(pipe_send, out_buf.c_str(), out_buf.size()); - if (bytes_written != static_cast(out_buf.size())) { - throw Error("BlackBoxExec", - "Failed to write the full request to the blackbox process."); - } - - // Read from process output pipe - char *str = NULL; - size_t size = 0; - - if (getline(&str, &size, file_receive) == -1) { - throw Error( - "BlackBoxExec", - "Reading blackbox process output from pipe resulted in error no. " + - std::to_string(errno)); - } - std::string in_buffer(str); - free(str); -#endif + std::string in_buffer = session().run(out_buf); // Parse the response in a single left-to-right pass: comma-separated // integers, a semicolon, then comma-separated floats (e.g. "5,-7;2.5,1.125\n"). const char *p = in_buffer.c_str(); @@ -328,19 +966,31 @@ void BlackBoxExec::run(const std::vector &int_in, ++q; } }; + auto skip_final_ws = [](const char *&q) { + while (*q == ' ' || *q == '\t' || *q == '\r' || *q == '\n') { + ++q; + } + }; for (size_t i = 0; i < int_out.size(); ++i) { + skip_ws(p); + errno = 0; long long v = std::strtoll(p, &end, 10); - if (end == p) { + if ((end == p) || (errno == ERANGE)) { throw Error("BlackBoxExec", "Failed to read output integer " + std::to_string(i) + " from blackbox process output, " + std::to_string(int_out.size()) + - " integer values where expected."); + " integer values were expected."); } - int_out[i] = static_cast(v); + int_out[i] = checked_int(v, "blackbox process output", i); p = end; skip_ws(p); - if (*p == ',') { + if (i + 1 < int_out.size()) { + if (*p != ',') { + throw Error("BlackBoxExec", + "Blackbox process response is missing an integer output " + "separator."); + } ++p; } } @@ -352,21 +1002,33 @@ void BlackBoxExec::run(const std::vector &int_in, } ++p; for (size_t i = 0; i < float_out.size(); ++i) { + skip_ws(p); + errno = 0; double v = std::strtod(p, &end); - if (end == p) { + if ((end == p) || (errno == ERANGE)) { throw Error("BlackBoxExec", "Failed to read output float " + std::to_string(i) + " from blackbox process output, " + std::to_string(float_out.size()) + - " floating point values where expected."); + " floating point values were expected."); } float_out[i] = v; p = end; skip_ws(p); - if (*p == ',') { + if (i + 1 < float_out.size()) { + if (*p != ',') { + throw Error("BlackBoxExec", + "Blackbox process response is missing a floating point " + "output separator."); + } ++p; } } + skip_final_ws(p); + if (*p != '\0') { + throw Error("BlackBoxExec", + "Blackbox process response contains trailing data."); + } } ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { @@ -503,11 +1165,30 @@ static void reason_subscriptions(const std::vector &reason, int n_int, return; } + const int n_total = n_int + n_float; size_t pos = 0; + auto require = [&](size_t n, const char *what) { + if (reason.size() - pos < n) { + throw Error("Blackbox", std::string("Malformed blackbox bounds reason: ") + + what + "."); + } + }; while (pos < reason.size()) { - pos++; // idx: the variable being explained (not needed for subscription) + require(1, "missing explained variable index"); + int idx = reason[pos++]; + if ((idx < 1) || (idx > n_total)) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: explained variable index " + "is out of range."); + } for (int side = 0; side < 2; side++) { // lower- then upper-bound literals + require(1, "missing reason literal count"); int count = reason[pos++]; + if (count < 0) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: negative literal count."); + } + require(static_cast(count) * 2, "truncated reason literals"); for (int k = 0; k < count; k++) { int var = reason[pos++]; // 1-based combined variable index pos++; // bound code (per-variable granularity only) @@ -515,6 +1196,10 @@ static void reason_subscriptions(const std::vector &reason, int n_int, sub_int[var - 1] = true; } else if (var > n_int && var <= n_int + n_float) { sub_float[var - 1 - n_int] = true; + } else { + throw Error("Blackbox", + "Malformed blackbox bounds reason: dependency variable " + "index is out of range."); } } } diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index 5e7bf5edd7..3e6c4c2b52 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -46,19 +47,24 @@ #endif #ifdef _WIN32 -#define NOMINMAX // Ensure the words min/max remain available -#include +#define GECODE_BLACKBOX_CALL __stdcall #else -// NOLINTNEXTLINE(bugprone-reserved-identifier) -#define __stdcall +#define GECODE_BLACKBOX_CALL #endif namespace Gecode { namespace FlatZinc { -/// Abstract class implemented by different methods to run blackbox functions +/// Abstract class implemented by different methods to run blackbox functions. +/// +/// A blackbox function must be deterministic in the FlatZinc sense: the same +/// integer and float inputs must always produce the same integer and float +/// outputs. Implementations may keep internal caches or other private state, +/// provided that this state does not make the observable result depend on call +/// order. class BlackBoxFn : public SharedHandle::Object { public: + virtual ~BlackBoxFn(void) {} virtual void run(const std::vector &int_in, const std::vector &float_in, std::vector &int_out, @@ -67,6 +73,9 @@ public: /// Implementation of a black box function that dynamically loads a library and /// run a contained function. +/// +/// The DLL entry points can be called concurrently by parallel search workers +/// and must therefore be thread-safe. class BlackBoxDLL : public BlackBoxFn { public: BlackBoxDLL(const std::string &name, const std::vector &args); @@ -81,12 +90,17 @@ public: protected: void *library; - void(__stdcall *dll_fzn_blackbox)(const int64_t *, size_t, const double *, - size_t, int64_t *, size_t, double *, size_t); + void(GECODE_BLACKBOX_CALL *dll_fzn_blackbox)( + const int64_t *, size_t, const double *, size_t, int64_t *, size_t, + double *, size_t); }; -/// Implementation of a black function that starts a seperate process to -/// repeatedly run a blackbox function, communication I/O over pipe. +/// Implementation of a blackbox function that starts a separate process to +/// repeatedly run a blackbox function, communicating over standard I/O. +/// +/// Parallel search workers do not share a process stream: each calling thread +/// gets its own persistent process session, created lazily and reused by that +/// thread until the shared blackbox object is destroyed. class BlackBoxExec : public BlackBoxFn { public: BlackBoxExec(const std::string &program, const std::vector &args); @@ -96,13 +110,18 @@ public: std::vector &float_out) override; protected: -#ifdef _WIN32 - HANDLE pipe_send; - HANDLE pipe_receive; -#else - int pipe_send; - FILE *file_receive; -#endif + class Session; + + /// The executable to run for each worker-thread session + std::string program; + /// Arguments passed to each executable session + std::vector args; + /// Mutex protecting the session table + Support::Mutex mutex; + /// One persistent process session for each calling thread + std::vector sessions; + + Session &session(void); }; class BlackBoxHandle : public SharedHandle { @@ -123,9 +142,11 @@ protected: ViewArray int_output; #ifdef GECODE_HAS_FLOAT_VARS - /// Floating-point variables considered as the integer input to the blackbox function + /// Floating-point variables considered as the floating-point input to the + /// blackbox function ViewArray float_input; - /// Floating-point variables set to the integer output of the blackbox function + /// Floating-point variables set to the floating-point output of the blackbox + /// function ViewArray float_output; #endif @@ -154,16 +175,17 @@ public: ViewArray &float_in, ViewArray &float_out, #endif - BlackBoxFn *black_box) + const BlackBoxHandle &black_box0) : Propagator(home), int_input(int_in), int_output(int_out), #ifdef GECODE_HAS_FLOAT_VARS float_input(float_in), float_output(float_out), #endif - black_box(black_box) { + black_box(black_box0) { int_input.subscribe(home, *this, Int::PC_INT_VAL); #ifdef GECODE_HAS_FLOAT_VARS float_input.subscribe(home, *this, Float::PC_FLOAT_VAL); #endif + home.notice(*this, AP_DISPOSE); } /// Cost function (defined as exponential) PropCost cost(const Space &home, const ModEventDelta &med) const override { @@ -175,9 +197,9 @@ public: }; /// Schedule function void reschedule(Space &home) override { - int_input.cancel(home, *this, Int::PC_INT_VAL); + int_input.reschedule(home, *this, Int::PC_INT_VAL); #ifdef GECODE_HAS_FLOAT_VARS - float_input.cancel(home, *this, Float::PC_FLOAT_VAL); + float_input.reschedule(home, *this, Float::PC_FLOAT_VAL); #endif } /// Delete propagator and return its size @@ -186,8 +208,9 @@ public: #ifdef GECODE_HAS_FLOAT_VARS float_input.cancel(home, *this, Float::PC_FLOAT_VAL); #endif + home.ignore(*this, AP_DISPOSE); + black_box.~BlackBoxHandle(); (void)Propagator::dispose(home); - // destroy plugin container return sizeof(*this); }; @@ -215,11 +238,12 @@ public: throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); } + BlackBoxHandle black_box_handle(black_box); new (home) BlackBox(home, int_input, int_output, #ifdef GECODE_HAS_FLOAT_VARS float_input, float_output, #endif - black_box); + black_box_handle); return ES_OK; } }; @@ -275,7 +299,7 @@ public: #ifdef GECODE_HAS_FLOAT_VARS SharedArray sub_float0, #endif - BlackBoxFn *black_box) + const BlackBoxHandle &black_box0) : Propagator(home), ivar(ivar), #ifdef GECODE_HAS_FLOAT_VARS fvar(fvar), @@ -284,7 +308,7 @@ public: #ifdef GECODE_HAS_FLOAT_VARS sub_float(sub_float0), #endif - black_box(black_box) { + black_box(black_box0) { for (int i = 0; i < ivar.size(); i++) { if (sub_int[i]) { ivar[i].subscribe(home, *this, Int::PC_INT_BND); @@ -297,6 +321,7 @@ public: } } #endif + home.notice(*this, AP_DISPOSE); } /// Cost function (defined as exponential) PropCost cost(const Space &home, const ModEventDelta &med) const override { @@ -334,9 +359,14 @@ public: fvar[i].cancel(home, *this, Float::PC_FLOAT_BND); } } +#endif + home.ignore(*this, AP_DISPOSE); + black_box.~BlackBoxHandle(); + sub_int.~SharedArray(); +#ifdef GECODE_HAS_FLOAT_VARS + sub_float.~SharedArray(); #endif (void)Propagator::dispose(home); - // destroy plugin container return sizeof(*this); }; @@ -366,6 +396,7 @@ public: throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); } + BlackBoxHandle black_box_handle(black_box); new (home) BlackBoxBounds(home, ivar, #ifdef GECODE_HAS_FLOAT_VARS fvar, @@ -374,7 +405,7 @@ public: #ifdef GECODE_HAS_FLOAT_VARS sub_float, #endif - black_box); + black_box_handle); return ES_OK; } }; From 84c9276b75e1c9490bdc12cd3da466f2089c8d7d Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Fri, 10 Jul 2026 20:53:02 +0200 Subject: [PATCH 03/14] Refine blackbox interfaces and backend structure Add the MiniZinc annotations, use platform-neutral backend naming, and separate process startup paths and library fallback handling. --- gecode/flatzinc/blackbox.cpp | 471 +++++++++--------- gecode/flatzinc/blackbox.hh | 41 +- .../blackbox/blackbox_annotations.mzn | 5 + .../experimental/blackbox/fzn_blackbox.mzn | 2 + .../blackbox/fzn_blackbox_bounds.mzn | 6 +- 5 files changed, 276 insertions(+), 249 deletions(-) create mode 100644 gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp index e9419ad3d4..eb7e3fbbda 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox.cpp @@ -47,7 +47,9 @@ #include #ifdef _WIN32 -#define NOMINMAX // Ensure the words min/max remain available +#ifndef NOMINMAX +#define NOMINMAX 1 // Ensure the words min/max remain available +#endif #include #else #include @@ -228,16 +230,17 @@ checked_int(long long v, const char *source, size_t i) { } // namespace -BlackBoxDLL::BlackBoxDLL(const std::string &name, - const std::vector &args) - : library(nullptr), dll_fzn_blackbox(nullptr) { +BlackBoxLibrary::BlackBoxLibrary(const std::string &name, + const std::vector &args) + : library(nullptr), library_fzn_blackbox(nullptr) { std::string loadError; void *loaded = nullptr; #ifdef _WIN32 + DWORD err = 0; std::wstring wname = utf8_to_wide(name); loaded = LoadLibraryW(wname.c_str()); if (!loaded) { - DWORD err = GetLastError(); + err = GetLastError(); loadError = std::string("unable to locate library `") + name + "'"; std::wstring wdll = utf8_to_wide(name + ".dll"); loaded = LoadLibraryW(wdll.c_str()); @@ -276,27 +279,27 @@ BlackBoxDLL::BlackBoxDLL(const std::string &name, // find symbol for blackbox function #ifdef _WIN32 - dll_fzn_blackbox = reinterpret_cast( + library_fzn_blackbox = reinterpret_cast( GetProcAddress((HMODULE)loaded, "fzn_blackbox")); #if defined(_M_IX86) || defined(__i386__) - if (!dll_fzn_blackbox) { - dll_fzn_blackbox = reinterpret_cast( + if (!library_fzn_blackbox) { + library_fzn_blackbox = reinterpret_cast( GetProcAddress((HMODULE)loaded, "_fzn_blackbox@32")); } - if (!dll_fzn_blackbox) { - dll_fzn_blackbox = reinterpret_cast( + if (!library_fzn_blackbox) { + library_fzn_blackbox = reinterpret_cast( GetProcAddress((HMODULE)loaded, "fzn_blackbox@32")); } #endif std::string symError = "."; #else - *(void **)(&dll_fzn_blackbox) = dlsym(loaded, "fzn_blackbox"); + *(void **)(&library_fzn_blackbox) = dlsym(loaded, "fzn_blackbox"); std::string symError(": "); - if (!dll_fzn_blackbox) { + if (!library_fzn_blackbox) { symError += std::string(dlerror()); } #endif - if (!dll_fzn_blackbox) { + if (!library_fzn_blackbox) { close_library(loaded); throw Error("Blackbox", "Unable to find symbol `fzn_blackbox` in dynamic library" + @@ -339,10 +342,23 @@ BlackBoxDLL::BlackBoxDLL(const std::string &name, library = loaded; } -BlackBoxDLL::~BlackBoxDLL() { +BlackBoxLibrary::~BlackBoxLibrary() { close_library(library); } +void +BlackBoxLibrary::run(const std::vector &int_in, + const std::vector &float_in, + std::vector &int_out, + std::vector &float_out) { + library_fzn_blackbox(int_in.data(), int_in.size(), float_in.data(), + float_in.size(), int_out.data(), int_out.size(), + float_out.data(), float_out.size()); + for (size_t i = 0; i < int_out.size(); ++i) { + int_out[i] = checked_int(int_out[i], "library output", i); + } +} + class BlackBoxExec::Session { protected: #ifdef _WIN32 @@ -368,6 +384,13 @@ class BlackBoxExec::Session { } #ifdef _WIN32 + static void close_handle(HANDLE &h) { + if (h != NULL) { + CloseHandle(h); + h = NULL; + } + } + static std::wstring quote_argument(const std::wstring &arg) { std::wstring q(L"\""); unsigned int backslashes = 0; @@ -388,6 +411,10 @@ class BlackBoxExec::Session { q += L'"'; return q; } + + void open_windows(const std::string &program, + const std::vector &args); + void close_windows(void); #else static bool reap_child(pid_t pid, int &status) { pid_t r; @@ -430,6 +457,10 @@ class BlackBoxExec::Session { } } while (errno == EINTR); } + + void open_posix(const std::string &program, + const std::vector &args); + void close_posix(void); #endif public: @@ -451,6 +482,110 @@ class BlackBoxExec::Session { #endif { #ifdef _WIN32 + open_windows(program, args); +#else + open_posix(program, args); +#endif + } + + ~Session(void) { close(); } + + bool owned_by_current_thread(void) const { +#ifdef GECODE_HAS_THREADS + return owner == std::this_thread::get_id(); +#else + return true; +#endif + } + + std::string run(const std::string &out_buf) { +#ifdef _WIN32 + size_t written = 0; + while (written < out_buf.size()) { + DWORD count = 0; + DWORD remaining = + static_cast(out_buf.size() - written); + BOOL success = + WriteFile(pipe_send, out_buf.data() + written, remaining, &count, + nullptr); + if (!success || count == 0) { + throw Error("BlackBoxExec", + last_error("Writing blackbox process input failed")); + } + written += count; + } + + char c[2] = {0, 0}; + std::ostringstream oss; + while (c[0] != '\n') { + DWORD count = 0; + BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); + if (!success) { + throw Error( + "BlackBoxExec", + "Failed to read blackbox process output from pipe"); + } else if (count == 0) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + assert(count == 1); + oss << c[0]; + } + return oss.str(); +#else + const char *p = out_buf.c_str(); + size_t remaining = out_buf.size(); + while (remaining > 0) { + ssize_t n = send_no_sigpipe(pipe_send, p, remaining); + if (n < 0) { + if (errno == EINTR) { + continue; + } + throw Error("BlackBoxExec", + "Writing blackbox process input failed with errno " + + std::to_string(errno)); + } + if (n == 0) { + throw Error("BlackBoxExec", + "Writing blackbox process input wrote zero bytes"); + } + p += n; + remaining -= static_cast(n); + } + + char *str = NULL; + size_t size = 0; + errno = 0; + if (getline(&str, &size, file_receive) == -1) { + free(str); + if (feof(file_receive)) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + throw Error( + "BlackBoxExec", + "Reading blackbox process output from pipe failed with errno " + + std::to_string(errno)); + } + std::string in_buffer(str); + free(str); + return in_buffer; +#endif + } + + void close(void) { +#ifdef _WIN32 + close_windows(); +#else + close_posix(); +#endif + } +}; + +#ifdef _WIN32 +void +BlackBoxExec::Session::open_windows(const std::string &program, + const std::vector &args) { // Build the command line before opening OS handles so allocation/conversion // failures cannot leak partially constructed process state. std::wstring program_w = utf8_to_wide(program); @@ -471,51 +606,55 @@ class BlackBoxExec::Session { saAttr.bInheritHandle = TRUE; saAttr.lpSecurityDescriptor = NULL; - HANDLE g_hChildStd_IN_Rd = NULL; - HANDLE g_hChildStd_IN_Wr = NULL; - HANDLE g_hChildStd_OUT_Rd = NULL; - HANDLE g_hChildStd_OUT_Wr = NULL; - HANDLE g_hChildStd_ERR_Wr = NULL; + HANDLE child_stdin_read = NULL; + HANDLE child_stdin_write = NULL; + HANDLE child_stdout_read = NULL; + HANDLE child_stdout_write = NULL; + HANDLE child_stderr_write = NULL; + LPPROC_THREAD_ATTRIBUTE_LIST attr_list = NULL; + + auto close_startup_handles = [&]() { + close_handle(child_stdin_read); + close_handle(child_stdin_write); + close_handle(child_stdout_read); + close_handle(child_stdout_write); + close_handle(child_stderr_write); + }; + auto destroy_attr_list = [&]() { + if (attr_list != NULL) { + DeleteProcThreadAttributeList(attr_list); + attr_list = NULL; + } + }; - // Create a pipe for the child process's STDOUT. - if (!CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0)) { + if (!CreatePipe(&child_stdout_read, &child_stdout_write, &saAttr, 0)) { throw Error("BlackBoxExec", last_error("Stdout CreatePipe failed")); } - // Ensure the read handle to the pipe for STDOUT is not inherited. - if (!SetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0)) { - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); + if (!SetHandleInformation(child_stdout_read, HANDLE_FLAG_INHERIT, 0)) { + DWORD err = GetLastError(); + close_startup_handles(); throw Error("BlackBoxExec", - last_error("Stdout SetHandleInformation failed")); + windows_error("Stdout SetHandleInformation failed", err)); } - - // Create a pipe for the child process's STDIN - if (!CreatePipe(&g_hChildStd_IN_Rd, &g_hChildStd_IN_Wr, &saAttr, 0)) { - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); - throw Error("BlackBoxExec", last_error("Stdin CreatePipe failed")); + if (!CreatePipe(&child_stdin_read, &child_stdin_write, &saAttr, 0)) { + DWORD err = GetLastError(); + close_startup_handles(); + throw Error("BlackBoxExec", windows_error("Stdin CreatePipe failed", err)); } - // Ensure the write handle to the pipe for STDIN is not inherited. - if (!SetHandleInformation(g_hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0)) { + if (!SetHandleInformation(child_stdin_write, HANDLE_FLAG_INHERIT, 0)) { DWORD err = GetLastError(); - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); - CloseHandle(g_hChildStd_IN_Rd); - CloseHandle(g_hChildStd_IN_Wr); - throw Error("BlackBoxExec", windows_error( - "Stdin SetHandleInformation failed", err)); + close_startup_handles(); + throw Error("BlackBoxExec", + windows_error("Stdin SetHandleInformation failed", err)); } HANDLE parent_stderr = GetStdHandle(STD_ERROR_HANDLE); if ((parent_stderr != NULL) && (parent_stderr != INVALID_HANDLE_VALUE)) { if (!DuplicateHandle(GetCurrentProcess(), parent_stderr, - GetCurrentProcess(), &g_hChildStd_ERR_Wr, 0, TRUE, + GetCurrentProcess(), &child_stderr_write, 0, TRUE, DUPLICATE_SAME_ACCESS)) { DWORD err = GetLastError(); - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); - CloseHandle(g_hChildStd_IN_Rd); - CloseHandle(g_hChildStd_IN_Wr); + close_startup_handles(); throw Error("BlackBoxExec", windows_error("stderr DuplicateHandle failed", err)); } @@ -523,52 +662,36 @@ class BlackBoxExec::Session { PROCESS_INFORMATION piProcInfo; STARTUPINFOEXW siStartInfo; - - // Set up members of the PROCESS_INFORMATION structure. ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); - - // Set up members of the STARTUPINFO structure. - // This structure specifies the STDIN and STDOUT handles for redirection. ZeroMemory(&siStartInfo, sizeof(STARTUPINFOEXW)); siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEXW); - siStartInfo.StartupInfo.hStdOutput = g_hChildStd_OUT_Wr; - siStartInfo.StartupInfo.hStdInput = g_hChildStd_IN_Rd; - siStartInfo.StartupInfo.hStdError = g_hChildStd_ERR_Wr; + siStartInfo.StartupInfo.hStdOutput = child_stdout_write; + siStartInfo.StartupInfo.hStdInput = child_stdin_read; + siStartInfo.StartupInfo.hStdError = child_stderr_write; siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; - HANDLE inherit_handles[3] = {g_hChildStd_IN_Rd, g_hChildStd_OUT_Wr, NULL}; + HANDLE inherit_handles[3] = {child_stdin_read, child_stdout_write, NULL}; DWORD inherit_count = 2; - if (g_hChildStd_ERR_Wr != NULL) { - inherit_handles[inherit_count++] = siStartInfo.StartupInfo.hStdError; + if (child_stderr_write != NULL) { + inherit_handles[inherit_count++] = child_stderr_write; } - siStartInfo.lpAttributeList = - reinterpret_cast(attr_buf.data()); - if (!InitializeProcThreadAttributeList(siStartInfo.lpAttributeList, 1, 0, - &attr_size)) { + attr_list = reinterpret_cast(attr_buf.data()); + siStartInfo.lpAttributeList = attr_list; + if (!InitializeProcThreadAttributeList(attr_list, 1, 0, &attr_size)) { DWORD err = GetLastError(); - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); - CloseHandle(g_hChildStd_IN_Rd); - CloseHandle(g_hChildStd_IN_Wr); - if (g_hChildStd_ERR_Wr != NULL) - CloseHandle(g_hChildStd_ERR_Wr); + close_startup_handles(); throw Error("BlackBoxExec", windows_error("InitializeProcThreadAttributeList failed", err)); } - if (!UpdateProcThreadAttribute(siStartInfo.lpAttributeList, 0, + if (!UpdateProcThreadAttribute(attr_list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, inherit_handles, sizeof(HANDLE) * inherit_count, NULL, NULL)) { DWORD err = GetLastError(); - DeleteProcThreadAttributeList(siStartInfo.lpAttributeList); - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); - CloseHandle(g_hChildStd_IN_Rd); - CloseHandle(g_hChildStd_IN_Wr); - if (g_hChildStd_ERR_Wr != NULL) - CloseHandle(g_hChildStd_ERR_Wr); + destroy_attr_list(); + close_startup_handles(); throw Error("BlackBoxExec", windows_error("PROC_THREAD_ATTRIBUTE_HANDLE_LIST failed", err)); } @@ -584,16 +707,11 @@ class BlackBoxExec::Session { nullptr, // use parent's current directory &siStartInfo.StartupInfo, &piProcInfo); // receives PROCESS_INFORMATION - DeleteProcThreadAttributeList(siStartInfo.lpAttributeList); + destroy_attr_list(); if (!processStarted) { DWORD err = GetLastError(); - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); - CloseHandle(g_hChildStd_IN_Rd); - CloseHandle(g_hChildStd_IN_Wr); - if (g_hChildStd_ERR_Wr != NULL) - CloseHandle(g_hChildStd_ERR_Wr); + close_startup_handles(); throw Error("BlackBoxExec", windows_error("Unable to start program `" + program + "'", err)); } @@ -622,30 +740,45 @@ class BlackBoxExec::Session { WaitForSingleObject(piProcInfo.hProcess, 5000); CloseHandle(piProcInfo.hThread); CloseHandle(piProcInfo.hProcess); - if (process_job != NULL) - CloseHandle(process_job); - CloseHandle(g_hChildStd_OUT_Rd); - CloseHandle(g_hChildStd_OUT_Wr); - CloseHandle(g_hChildStd_IN_Rd); - CloseHandle(g_hChildStd_IN_Wr); - if (g_hChildStd_ERR_Wr != NULL) - CloseHandle(g_hChildStd_ERR_Wr); + close_handle(process_job); + close_startup_handles(); throw Error("BlackBoxExec", windows_error("ResumeThread failed for blackbox process", err)); } CloseHandle(piProcInfo.hThread); - // Stop ReadFile from blocking - CloseHandle(g_hChildStd_OUT_Wr); - // Just close the child's in pipe here - CloseHandle(g_hChildStd_IN_Rd); - if (g_hChildStd_ERR_Wr != NULL) - CloseHandle(g_hChildStd_ERR_Wr); - - pipe_send = g_hChildStd_IN_Wr; - pipe_receive = g_hChildStd_OUT_Rd; + + close_handle(child_stdout_write); + close_handle(child_stdin_read); + close_handle(child_stderr_write); + + pipe_send = child_stdin_write; + pipe_receive = child_stdout_read; process = piProcInfo.hProcess; job = process_job; +} + +void +BlackBoxExec::Session::close_windows(void) { + close_handle(pipe_send); + close_handle(pipe_receive); + if (process != NULL) { + DWORD wait = WaitForSingleObject(process, 1000); + if (wait == WAIT_TIMEOUT) { + if (job != NULL) { + TerminateJobObject(job, 1); + } else { + TerminateProcess(process, 1); + } + WaitForSingleObject(process, 5000); + } + close_handle(process); + } + close_handle(job); +} #else +void +BlackBoxExec::Session::open_posix(const std::string &program, + const std::vector &args) { const int READ = 0; const int WRITE = 1; int child_in[2] = {-1, -1}; @@ -774,138 +907,24 @@ class BlackBoxExec::Session { child = -1; throw Error("BlackBoxExec", last_error("fdopen failed")); } - return; -#endif - } - - ~Session(void) { close(); } +} - bool owned_by_current_thread(void) const { -#ifdef GECODE_HAS_THREADS - return owner == std::this_thread::get_id(); -#else - return true; -#endif +void +BlackBoxExec::Session::close_posix(void) { + if (pipe_send != -1) { + ::close(pipe_send); + pipe_send = -1; } - - std::string run(const std::string &out_buf) { -#ifdef _WIN32 - size_t written = 0; - while (written < out_buf.size()) { - DWORD count = 0; - DWORD remaining = - static_cast(out_buf.size() - written); - BOOL success = - WriteFile(pipe_send, out_buf.data() + written, remaining, &count, - nullptr); - if (!success || count == 0) { - throw Error("BlackBoxExec", - last_error("Writing blackbox process input failed")); - } - written += count; - } - - char c[2] = {0, 0}; - std::ostringstream oss; - while (c[0] != '\n') { - DWORD count = 0; - BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); - if (!success) { - throw Error( - "BlackBoxExec", - "Reading blackbox process output from pipe resulted did not succeed"); - } else if (count == 0) { - throw Error("BlackBoxExec", - "Blackbox process provided an incomplete response"); - } - assert(count == 1); - oss << c[0]; - } - return oss.str(); -#else - const char *p = out_buf.c_str(); - size_t remaining = out_buf.size(); - while (remaining > 0) { - ssize_t n = send_no_sigpipe(pipe_send, p, remaining); - if (n < 0) { - if (errno == EINTR) { - continue; - } - throw Error("BlackBoxExec", - "Writing blackbox process input failed with errno " + - std::to_string(errno)); - } - if (n == 0) { - throw Error("BlackBoxExec", - "Writing blackbox process input wrote zero bytes"); - } - p += n; - remaining -= static_cast(n); - } - - char *str = NULL; - size_t size = 0; - errno = 0; - if (getline(&str, &size, file_receive) == -1) { - free(str); - if (feof(file_receive)) { - throw Error("BlackBoxExec", - "Blackbox process provided an incomplete response"); - } - throw Error( - "BlackBoxExec", - "Reading blackbox process output from pipe resulted in error no. " + - std::to_string(errno)); - } - std::string in_buffer(str); - free(str); - return in_buffer; -#endif + if (file_receive != NULL) { + fclose(file_receive); + file_receive = NULL; } - - void close(void) { -#ifdef _WIN32 - if (pipe_send != NULL) { - CloseHandle(pipe_send); - pipe_send = NULL; - } - if (pipe_receive != NULL) { - CloseHandle(pipe_receive); - pipe_receive = NULL; - } - if (process != NULL) { - DWORD wait = WaitForSingleObject(process, 1000); - if (wait == WAIT_TIMEOUT) { - if (job != NULL) { - TerminateJobObject(job, 1); - } else { - TerminateProcess(process, 1); - } - WaitForSingleObject(process, 5000); - } - CloseHandle(process); - process = NULL; - } - if (job != NULL) { - CloseHandle(job); - job = NULL; - } -#else - if (pipe_send != -1) { - ::close(pipe_send); - pipe_send = -1; - } - if (file_receive != NULL) { - fclose(file_receive); - file_receive = NULL; - } - if (child > 0) { - terminate_child(child); - child = -1; - } -#endif + if (child > 0) { + terminate_child(child); + child = -1; } -}; +} +#endif BlackBoxExec::BlackBoxExec(const std::string &program0, const std::vector &args0) diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index 3e6c4c2b52..ffabd7fd45 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -36,7 +36,6 @@ #include #include -#include #include #include @@ -74,23 +73,21 @@ public: /// Implementation of a black box function that dynamically loads a library and /// run a contained function. /// -/// The DLL entry points can be called concurrently by parallel search workers -/// and must therefore be thread-safe. -class BlackBoxDLL : public BlackBoxFn { +/// The native library entry points can be called concurrently by parallel +/// search workers and must therefore be thread-safe. +class BlackBoxLibrary : public BlackBoxFn { public: - BlackBoxDLL(const std::string &name, const std::vector &args); - ~BlackBoxDLL(); + BlackBoxLibrary(const std::string &name, + const std::vector &args); + ~BlackBoxLibrary(); void run(const std::vector &int_in, - const std::vector &float_in, std::vector &int_out, - std::vector &float_out) override { - dll_fzn_blackbox(int_in.data(), int_in.size(), float_in.data(), - float_in.size(), int_out.data(), int_out.size(), - float_out.data(), float_out.size()); - } + const std::vector &float_in, + std::vector &int_out, + std::vector &float_out) override; protected: void *library; - void(GECODE_BLACKBOX_CALL *dll_fzn_blackbox)( + void(GECODE_BLACKBOX_CALL *library_fzn_blackbox)( const int64_t *, size_t, const double *, size_t, int64_t *, size_t, double *, size_t); }; @@ -112,13 +109,13 @@ public: protected: class Session; - /// The executable to run for each worker-thread session + /// The executable to run for each worker-thread session. std::string program; - /// Arguments passed to each executable session + /// Arguments passed to each executable session. std::vector args; - /// Mutex protecting the session table + /// Mutex protecting the session table. Support::Mutex mutex; - /// One persistent process session for each calling thread + /// One persistent process session for each calling thread. std::vector sessions; Session &session(void); @@ -231,7 +228,7 @@ public: const std::vector &args) { BlackBoxFn *black_box(nullptr); if (mode == "dll") { - black_box = new BlackBoxDLL(instantiation, args); + black_box = new BlackBoxLibrary(instantiation, args); } else if (mode == "exec") { black_box = new BlackBoxExec(instantiation, args); } else { @@ -250,11 +247,13 @@ public: class BlackBoxBounds : public Propagator { protected: - /// Integer variables whose bounds are input and computed by the blackbox function (in order). + /// Integer variables whose bounds are input and computed by the blackbox + /// function, in order. ViewArray ivar; #ifdef GECODE_HAS_FLOAT_VARS - /// Floating-point variables whose bounds are input and computed by the blackbox function (in order). + /// Floating-point variables whose bounds are input and computed by the + /// blackbox function, in order. ViewArray fvar; #endif @@ -389,7 +388,7 @@ public: const std::vector &args) { BlackBoxFn *black_box(nullptr); if (mode == "dll") { - black_box = new BlackBoxDLL(instantiation, args); + black_box = new BlackBoxLibrary(instantiation, args); } else if (mode == "exec") { black_box = new BlackBoxExec(instantiation, args); } else { diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn new file mode 100644 index 0000000000..0f7588e078 --- /dev/null +++ b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn @@ -0,0 +1,5 @@ +annotation blackbox_dll(string: library); +annotation blackbox_dll(string: library, array[int] of string: args); + +annotation blackbox_exec(string: command); +annotation blackbox_exec(string: command, array[int] of string: args); diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn index c8d39f1c03..44407aedf8 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn @@ -1,3 +1,5 @@ +include "blackbox_annotations.mzn"; + predicate fzn_blackbox( array[int] of var int: int_input, array[int] of var float: float_input, diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn index af24151a7c..68fa9640d0 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn @@ -1,11 +1,13 @@ +include "blackbox_annotations.mzn"; + predicate fzn_blackbox_bounds( array[int] of var int: int_input, array[int] of var float: float_input, - array[int] of int: flat_reason, + array[int] of int: flat_reason ) = gecode_blackbox_bounds(int_input, float_input, flat_reason); predicate gecode_blackbox_bounds( array[int] of var int: int_input, array[int] of var float: float_input, - array[int] of int: flat_reason, + array[int] of int: flat_reason ); From ef44fb09b45bd1670e79d3938c337edceeef42c3 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Fri, 10 Jul 2026 20:53:02 +0200 Subject: [PATCH 04/14] Define and validate blackbox contracts Validate protocol data and posting rules, document the callback contract, and integrate generated-header dependencies with Autoconf. --- Makefile.in | 10 +- changelog.in | 6 +- cmake/GecodeSources.cmake | 1 + configure | 71 ++++++- configure.ac | 8 + gecode/flatzinc/blackbox.cpp | 168 +++++++++++---- gecode/flatzinc/blackbox.hh | 74 +++++++ .../blackbox/blackbox_annotations.mzn | 34 +++ .../experimental/blackbox/fzn_blackbox.mzn | 3 + .../blackbox/fzn_blackbox_bounds.mzn | 5 + gecode/flatzinc/registry.cpp | 45 +++- test/flatzinc.cpp | 40 ++++ test/flatzinc.hh | 8 + test/flatzinc/blackbox.cpp | 198 ++++++++++++++++++ 14 files changed, 611 insertions(+), 60 deletions(-) create mode 100644 test/flatzinc/blackbox.cpp diff --git a/Makefile.in b/Makefile.in index cb30b9b5ed..4e9654cc38 100755 --- a/Makefile.in +++ b/Makefile.in @@ -77,6 +77,7 @@ export QT_CPPFLAGS = @QTDEFINES@ @QTINCLUDES@ export LINKQT = @QTLIBS@ export LINKCPPROFILER = @LINKCPPROFILER@ export LINKATOMIC = @LINKATOMIC@ +export LINKDL = @GECODE_DL_LIBS@ export MPFR_CPPFLAGS = @GMP_CPPFLAGS@ @MPFR_CPPFLAGS@ ifeq "@enable_mpfr@" "yes" export LINKMPFR = @MPFR_LIB_PATH@ @GMP_LIB_PATH@ @MPFR_LINK@ @GMP_LINK@ @@ -1168,6 +1169,7 @@ FLATZINCTESTSRC0 = \ test/flatzinc/battleships5.cpp \ test/flatzinc/battleships7.cpp \ test/flatzinc/battleships9.cpp \ + test/flatzinc/blackbox.cpp \ test/flatzinc/blocksworld_instance_1.cpp \ test/flatzinc/blocksworld_instance_2.cpp \ test/flatzinc/cumulatives.cpp \ @@ -1395,6 +1397,8 @@ gecode/kernel/var-type.hpp: $(VISDEP) gecode/kernel/var-imp.hpp: $(VISDEP) $(UVRUN) $(top_srcdir)/misc/genvarimp.py -header $(VIS) > $@ +$(ALLOBJ) $(ALLSBJ) $(TESTOBJ) $(TESTSBJ): $(VARIMP) + # # Object targets # @@ -1566,7 +1570,7 @@ else export LINKALL = \ $(LINKFLATZINC) $(LINKDRIVER) $(LINKGIST) \ $(LINKSEARCH) $(LINKMM) $(LINKSET) $(LINKFLOAT) $(LINKMPFR) $(LINKINT) \ - $(LINKKERNEL) $(LINKSUPPORT) $(LINKATOMIC) + $(LINKKERNEL) $(LINKSUPPORT) $(LINKATOMIC) $(LINKDL) endif $(SUPPORTDLL): $(SUPPORTOBJ) @@ -1627,7 +1631,7 @@ $(FLATZINCDLL): $(FLATZINCOBJ) $(SUPPORTDLL) $(KERNELDLL) $(SEARCHDLL) \ $(CXX) $(DLLFLAGS) $(FLATZINCOBJ) $(FLATZINCSONAME) \ @DLLPATH@ $(LINKSUPPORT) $(LINKKERNEL) $(LINKSEARCH) $(LINKINT) \ $(LINKSET) $(LINKFLOAT) $(LINKMM) $(LINKGIST) $(LINKDRIVER) $(LINKQT) \ - @LINKOUTPUT@$(FLATZINCDLL) + $(LINKDL) @LINKOUTPUT@$(FLATZINCDLL) $(CREATELINK) $@ $(@:%$(DLLSUFFIX)=%$(SOLINKSUFFIX)) $(CREATELINK) $@ $(@:%$(DLLSUFFIX)=%$(SOSUFFIX)) else @@ -1735,7 +1739,7 @@ $(FLATZINCDLL) $(FLATZINCLIB): $(FLATZINCOBJ) $(FLATZINCRES) \ $(SUPPORTDLL) $(KERNELDLL) $(SEARCHDLL) $(INTDLL) \ $(SETDLL) $(FLOATDLL) $(GISTDLL) $(MMDLL) $(DRIVERDLL) $(CXX) $(DLLFLAGS) $(FLATZINCOBJ) $(FLATZINCRES) \ - @DLLPATH@ @LINKOUTPUT@$(FLATZINCDLL) $(GLDFLAGS) $(LINKQT) + @DLLPATH@ @LINKOUTPUT@$(FLATZINCDLL) $(GLDFLAGS) $(LINKQT) $(LINKDL) $(FIXMANIFEST) $(FLATZINCDLL).manifest $(MANIFEST) -manifest $(FLATZINCDLL).manifest \ -outputresource:$(FLATZINCDLL)\;2 diff --git a/changelog.in b/changelog.in index be3ec50ac7..9f77bd4519 100755 --- a/changelog.in +++ b/changelog.in @@ -84,9 +84,11 @@ Add support for the experimental MiniZinc black-box propagator interface. A FlatZinc model can request propagation using an external function, implemented either as a shared library or as a subprocess, through two generic propagators: gecode_blackbox (value propagation, scheduled once all inputs are fixed) and -gecode_blackbox_bounds (bounds propagation, scheduled on bound changes).The +gecode_blackbox_bounds (bounds propagation, scheduled on bound changes). The blackbox_exec and blackbox_dll annotations select the execution mode and pass -through extra arguments. +through extra arguments. These annotations intentionally execute user-provided +code and should only be used with trusted models and trusted executable or +library paths. [ENTRY] Module: minimodel diff --git a/cmake/GecodeSources.cmake b/cmake/GecodeSources.cmake index affe090e31..4251a0b84e 100644 --- a/cmake/GecodeSources.cmake +++ b/cmake/GecodeSources.cmake @@ -264,6 +264,7 @@ set(GECODE_TEST_SOURCES test/flatzinc/battleships5.cpp test/flatzinc/battleships7.cpp test/flatzinc/battleships9.cpp + test/flatzinc/blackbox.cpp test/flatzinc/blocksworld_instance_1.cpp test/flatzinc/blocksworld_instance_2.cpp test/flatzinc/bool_clause.cpp diff --git a/configure b/configure index 8aa3bf7718..084c071bce 100755 --- a/configure +++ b/configure @@ -37,7 +37,6 @@ esac fi - # Reset variables that may have inherited troublesome values from # the environment. @@ -665,6 +664,7 @@ enable_minimodel enable_examples enable_flatzinc enable_driver +GECODE_DL_LIBS host_os host_vendor host_cpu @@ -14458,6 +14458,75 @@ printf "%s\n" "#define HAVE_MMAP 1" >>confdefs.h fi rm -f conftest.mmap conftest.txt +GECODE_SAVE_LIBS=${LIBS} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5 +printf %s "checking for library containing dlopen... " >&6; } +if test ${ac_cv_search_dlopen+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +namespace conftest { + extern "C" int dlopen (); +} +int +main (void) +{ +return conftest::dlopen (); + ; + return 0; +} +_ACEOF +for ac_lib in '' dl +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_cxx_try_link "$LINENO" +then : + ac_cv_search_dlopen=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_dlopen+y} +then : + break +fi +done +if test ${ac_cv_search_dlopen+y} +then : + +else case e in #( + e) ac_cv_search_dlopen=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dlopen" >&5 +printf "%s\n" "$ac_cv_search_dlopen" >&6; } +ac_res=$ac_cv_search_dlopen +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + GECODE_DL_LIBS=${ac_cv_search_dlopen} +else case e in #( + e) GECODE_DL_LIBS= ;; +esac +fi + +LIBS=${GECODE_SAVE_LIBS} +if test "x${GECODE_DL_LIBS}" = "xnone required"; then + GECODE_DL_LIBS= +fi + # Check whether --enable-driver was given. diff --git a/configure.ac b/configure.ac index cbc447bd95..668cf8af68 100644 --- a/configure.ac +++ b/configure.ac @@ -279,6 +279,14 @@ AC_GECODE_CBS AC_GECODE_CPPROFILER AC_GECODE_FLEXBISON AC_FUNC_MMAP +GECODE_SAVE_LIBS=${LIBS} +AC_SEARCH_LIBS([dlopen], [dl], [GECODE_DL_LIBS=${ac_cv_search_dlopen}], + [GECODE_DL_LIBS=]) +LIBS=${GECODE_SAVE_LIBS} +if test "x${GECODE_DL_LIBS}" = "xnone required"; then + GECODE_DL_LIBS= +fi +AC_SUBST([GECODE_DL_LIBS]) AC_GECODE_ENABLE_MODULE(driver, yes, [build script commandline driver library], diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp index eb7e3fbbda..427c06aa0f 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox.cpp @@ -40,6 +40,8 @@ #include #include #include +#include +#include #include #include #include @@ -228,6 +230,39 @@ checked_int(long long v, const char *source, size_t i) { return static_cast(v); } +#ifdef GECODE_HAS_FLOAT_VARS +void +check_float(double v, const char *source, size_t i) { + static_assert(sizeof(double) == sizeof(std::uint64_t) && + std::numeric_limits::is_iec559, + "blackbox floats must use IEEE-754 binary64"); + std::uint64_t bits = 0; + const volatile unsigned char *raw = + reinterpret_cast(&v); + unsigned char *target = reinterpret_cast(&bits); + for (size_t j = 0; j < sizeof(bits); j++) { + target[j] = raw[j]; + } + if (((bits & UINT64_C(0x7ff0000000000000)) == + UINT64_C(0x7ff0000000000000)) || + (v < Float::Limits::min) || (v > Float::Limits::max)) { + throw Error("Blackbox", std::string(source) + " float " + + std::to_string(i) + + " is not a finite value in Gecode's floating " + "point range"); + } +} + +void +check_floats(const std::vector &v, const char *source) { + for (size_t i = 0; i < v.size(); i++) { + check_float(v[i], source, i); + } +} +#endif + +const size_t max_exec_response_size = 1024 * 1024; + } // namespace BlackBoxLibrary::BlackBoxLibrary(const std::string &name, @@ -357,6 +392,9 @@ BlackBoxLibrary::run(const std::vector &int_in, for (size_t i = 0; i < int_out.size(); ++i) { int_out[i] = checked_int(int_out[i], "library output", i); } +#ifdef GECODE_HAS_FLOAT_VARS + check_floats(float_out, "library output"); +#endif } class BlackBoxExec::Session { @@ -517,6 +555,7 @@ class BlackBoxExec::Session { char c[2] = {0, 0}; std::ostringstream oss; + size_t response_size = 0; while (c[0] != '\n') { DWORD count = 0; BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); @@ -529,6 +568,10 @@ class BlackBoxExec::Session { "Blackbox process provided an incomplete response"); } assert(count == 1); + if (++response_size > max_exec_response_size) { + throw Error("BlackBoxExec", + "Blackbox process response exceeds the size limit"); + } oss << c[0]; } return oss.str(); @@ -553,22 +596,29 @@ class BlackBoxExec::Session { remaining -= static_cast(n); } - char *str = NULL; - size_t size = 0; - errno = 0; - if (getline(&str, &size, file_receive) == -1) { - free(str); - if (feof(file_receive)) { + std::string in_buffer; + while (true) { + errno = 0; + int ch = fgetc(file_receive); + if (ch == EOF) { + if (feof(file_receive)) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } throw Error("BlackBoxExec", - "Blackbox process provided an incomplete response"); + std::string("Reading blackbox process output from pipe " + "failed with errno ") + + std::to_string(errno)); + } + in_buffer += static_cast(ch); + if (in_buffer.size() > max_exec_response_size) { + throw Error("BlackBoxExec", + "Blackbox process response exceeds the size limit"); + } + if (ch == '\n') { + break; } - throw Error( - "BlackBoxExec", - "Reading blackbox process output from pipe failed with errno " + - std::to_string(errno)); } - std::string in_buffer(str); - free(str); return in_buffer; #endif } @@ -958,7 +1008,8 @@ void BlackBoxExec::run(const std::vector &int_in, std::vector &float_out) { // Construct program input: comma-separated integers, a semicolon, then // comma-separated floats, terminated by a newline (e.g. "5,-7;2.5,1.125\n"). - std::stringstream out; + std::ostringstream out; + out.imbue(std::locale::classic()); out.precision(std::numeric_limits::max_digits10); for (size_t i = 0; i < int_in.size(); ++i) { if (i != 0) { @@ -976,10 +1027,13 @@ void BlackBoxExec::run(const std::vector &int_in, out << "\n"; std::string out_buf = out.str(); std::string in_buffer = session().run(out_buf); + if (in_buffer.find('\0') != std::string::npos) { + throw Error("BlackBoxExec", + "Blackbox process response contains NUL data."); + } // Parse the response in a single left-to-right pass: comma-separated // integers, a semicolon, then comma-separated floats (e.g. "5,-7;2.5,1.125\n"). const char *p = in_buffer.c_str(); - char *end = nullptr; auto skip_ws = [](const char *&q) { while (*q == ' ' || *q == '\t' || *q == '\r') { ++q; @@ -990,11 +1044,28 @@ void BlackBoxExec::run(const std::vector &int_in, ++q; } }; + auto value_end = [](const char *q) { + while (*q != ',' && *q != ';' && *q != '\n' && *q != '\0') { + ++q; + } + return q; + }; + auto check_number_tail = [](std::istringstream &in) { + char c; + while (in.get(c)) { + if (c != ' ' && c != '\t' && c != '\r') { + return false; + } + } + return true; + }; for (size_t i = 0; i < int_out.size(); ++i) { skip_ws(p); - errno = 0; - long long v = std::strtoll(p, &end, 10); - if ((end == p) || (errno == ERANGE)) { + const char *end = value_end(p); + std::istringstream in(std::string(p, end)); + in.imbue(std::locale::classic()); + long long v; + if (!(in >> v) || !check_number_tail(in)) { throw Error("BlackBoxExec", "Failed to read output integer " + std::to_string(i) + " from blackbox process output, " + @@ -1022,15 +1093,20 @@ void BlackBoxExec::run(const std::vector &int_in, ++p; for (size_t i = 0; i < float_out.size(); ++i) { skip_ws(p); - errno = 0; - double v = std::strtod(p, &end); - if ((end == p) || (errno == ERANGE)) { + const char *end = value_end(p); + std::istringstream in(std::string(p, end)); + in.imbue(std::locale::classic()); + double v; + if (!(in >> v) || !check_number_tail(in)) { throw Error("BlackBoxExec", "Failed to read output float " + std::to_string(i) + " from blackbox process output, " + std::to_string(float_out.size()) + " floating point values were expected."); } +#ifdef GECODE_HAS_FLOAT_VARS + check_float(v, "blackbox process output", i); +#endif float_out[i] = v; p = end; skip_ws(p); @@ -1060,7 +1136,6 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { std::vector int_out(int_output.size()); // std::cerr << "Black Box Fn input: "; for (int i = 0; i < int_in.size(); i++) { - // std::cerr << int_input[i].val() << " "; int_in[i] = int_input[i].val(); } std::vector float_in; @@ -1069,26 +1144,21 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { float_in.resize(float_input.size()); float_out.resize(float_output.size()); for (int i = 0; i < float_in.size(); i++) { - // std::cerr << float_input[i].val() << " "; float_in[i] = float_input[i].val().med(); } #endif - // std::cerr << std::endl; black_box()->run(int_in, float_in, int_out, float_out); - // std::cerr << "Black Box Fn output: "; for (int i = 0; i < int_out.size(); i++) { // std::cerr << int_out[i] << " "; GECODE_ME_CHECK(int_output[i].eq(home, static_cast(int_out[i]))); } #ifdef GECODE_HAS_FLOAT_VARS for (int i = 0; i < float_out.size(); i++) { - // std::cerr << float_out[i] << " "; GECODE_ME_CHECK(float_output[i].eq(home, float_out[i])); } #endif - // std::cerr << std::endl; return home.ES_SUBSUMED(*this); } @@ -1100,7 +1170,6 @@ ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { std::vector int_out(ivar.size() * 2); // std::cerr << "Black Box Bounds Fn input: "; for (int i = 0; i < ivar.size(); i++) { - // std::cerr << ivar[i].min() << " " << ivar[i].max() << " "; int_in[i*2] = ivar[i].min(); int_in[i*2+1] = ivar[i].max(); } @@ -1110,16 +1179,13 @@ ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { float_in.resize(fvar.size() * 2); float_out.resize(fvar.size() * 2); for (int i = 0; i < fvar.size(); i++) { - // std::cerr << fvar[i].min() << " " << fvar[i].max() << " "; float_in[i*2] = fvar[i].min(); float_in[i*2+1] = fvar[i].max(); } #endif - // std::cerr << std::endl; black_box()->run(int_in, float_in, int_out, float_out); - // std::cerr << "Black Box Fn output: "; for (int i = 0; i < ivar.size(); i++) { // std::cerr << int_out[i*2] << ".." << int_out[i*2+1] << " "; GECODE_ME_CHECK(ivar[i].gq(home, static_cast(int_out[i*2]))); @@ -1127,12 +1193,10 @@ ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { } #ifdef GECODE_HAS_FLOAT_VARS for (int i = 0; i < fvar.size(); i++) { - // std::cerr << float_out[i*2] << ".." << float_out[i*2+1] << " "; GECODE_ME_CHECK(fvar[i].gq(home, float_out[i*2])); GECODE_ME_CHECK(fvar[i].lq(home, float_out[i*2+1])); } #endif - // std::cerr << std::endl; return ES_NOFIX; } @@ -1168,23 +1232,19 @@ void blackbox(Home home, const IntVarArgs &int_in, const IntVarArgs &int_out, /// variables first, then float variables. /// /// The flat reason is a concatenation of one entry per variable, each entry -/// being `[idx, |R_lb|, (var, bnd)..., |R_ub|, (var, bnd)...]`. An empty reason -/// falls back to subscribing to every variable. +/// being `[idx, |R_lb|, (var, bnd)..., |R_ub|, (var, bnd)...]`. static void reason_subscriptions(const std::vector &reason, int n_int, int n_float, SharedArray &sub_int, SharedArray &sub_float) { - const bool all = reason.empty(); for (int i = 0; i < n_int; i++) { - sub_int[i] = all; + sub_int[i] = false; } for (int i = 0; i < n_float; i++) { - sub_float[i] = all; - } - if (all) { - return; + sub_float[i] = false; } const int n_total = n_int + n_float; + std::vector explained(n_total, false); size_t pos = 0; auto require = [&](size_t n, const char *what) { if (reason.size() - pos < n) { @@ -1200,6 +1260,12 @@ static void reason_subscriptions(const std::vector &reason, int n_int, "Malformed blackbox bounds reason: explained variable index " "is out of range."); } + if (explained[idx - 1]) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: duplicate explained " + "variable index."); + } + explained[idx - 1] = true; for (int side = 0; side < 2; side++) { // lower- then upper-bound literals require(1, "missing reason literal count"); int count = reason[pos++]; @@ -1207,10 +1273,14 @@ static void reason_subscriptions(const std::vector &reason, int n_int, throw Error("Blackbox", "Malformed blackbox bounds reason: negative literal count."); } - require(static_cast(count) * 2, "truncated reason literals"); + if (static_cast(count) > (reason.size() - pos) / 2) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: truncated reason " + "literals."); + } for (int k = 0; k < count; k++) { int var = reason[pos++]; // 1-based combined variable index - pos++; // bound code (per-variable granularity only) + int bnd = reason[pos++]; if (var >= 1 && var <= n_int) { sub_int[var - 1] = true; } else if (var > n_int && var <= n_int + n_float) { @@ -1220,9 +1290,21 @@ static void reason_subscriptions(const std::vector &reason, int n_int, "Malformed blackbox bounds reason: dependency variable " "index is out of range."); } + if (bnd != 1 && bnd != 2) { // MiniZinc PropBnd: PR_LB, PR_UB + throw Error("Blackbox", + "Malformed blackbox bounds reason: dependency bound " + "code is out of range."); + } } } } + for (int i = 0; i < n_total; i++) { + if (!explained[i]) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: missing explained " + "variable entry."); + } + } } void blackbox_bounds(Home home, const IntVarArgs &ivar, diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index ffabd7fd45..14b8b2163c 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -236,6 +236,34 @@ public: } BlackBoxHandle black_box_handle(black_box); + if ((int_input.size() == 0) +#ifdef GECODE_HAS_FLOAT_VARS + && (float_input.size() == 0) +#endif + ) { + std::vector int_in; + std::vector int_out(int_output.size()); + std::vector float_in; + std::vector float_out; +#ifdef GECODE_HAS_FLOAT_VARS + float_out.resize(float_output.size()); +#endif + black_box_handle()->run(int_in, float_in, int_out, float_out); + for (int i = 0; i < int_output.size(); i++) { + if (me_failed(int_output[i].eq(home, int_out[i]))) { + return ES_FAILED; + } + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < float_output.size(); i++) { + if (me_failed(float_output[i].eq(home, float_out[i]))) { + return ES_FAILED; + } + } +#endif + return ES_OK; + } + new (home) BlackBox(home, int_input, int_output, #ifdef GECODE_HAS_FLOAT_VARS float_input, float_output, @@ -321,6 +349,7 @@ public: } #endif home.notice(*this, AP_DISPOSE); + home.notice(*this, AP_WEAKLY); } /// Cost function (defined as exponential) PropCost cost(const Space &home, const ModEventDelta &med) const override { @@ -360,6 +389,7 @@ public: } #endif home.ignore(*this, AP_DISPOSE); + home.ignore(*this, AP_WEAKLY); black_box.~BlackBoxHandle(); sub_int.~SharedArray(); #ifdef GECODE_HAS_FLOAT_VARS @@ -396,6 +426,50 @@ public: } BlackBoxHandle black_box_handle(black_box); + bool has_subscription = false; + for (int i = 0; i < ivar.size(); i++) { + has_subscription = has_subscription || sub_int[i]; + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) { + has_subscription = has_subscription || sub_float[i]; + } +#endif + if (!has_subscription) { + std::vector int_in(ivar.size() * 2); + std::vector int_out(ivar.size() * 2); + for (int i = 0; i < ivar.size(); i++) { + int_in[i*2] = ivar[i].min(); + int_in[i*2+1] = ivar[i].max(); + } + std::vector float_in; + std::vector float_out; +#ifdef GECODE_HAS_FLOAT_VARS + float_in.resize(fvar.size() * 2); + float_out.resize(fvar.size() * 2); + for (int i = 0; i < fvar.size(); i++) { + float_in[i*2] = fvar[i].min(); + float_in[i*2+1] = fvar[i].max(); + } +#endif + black_box_handle()->run(int_in, float_in, int_out, float_out); + for (int i = 0; i < ivar.size(); i++) { + if (me_failed(ivar[i].gq(home, int_out[i*2])) || + me_failed(ivar[i].lq(home, int_out[i*2+1]))) { + return ES_FAILED; + } + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) { + if (me_failed(fvar[i].gq(home, float_out[i*2])) || + me_failed(fvar[i].lq(home, float_out[i*2+1]))) { + return ES_FAILED; + } + } +#endif + return ES_OK; + } + new (home) BlackBoxBounds(home, ivar, #ifdef GECODE_HAS_FLOAT_VARS fvar, diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn index 0f7588e078..9b5f8df903 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn @@ -1,3 +1,37 @@ +% Blackbox annotations execute user-provided code. Use them only for trusted +% models and trusted executable/library paths. +% +% blackbox_dll loads a native library and calls: +% +% void fzn_blackbox( +% const int64_t* int_in, size_t n_int_in, +% const double* float_in, size_t n_float_in, +% int64_t* int_out, size_t n_int_out, +% double* float_out, size_t n_float_out) +% +% The library may also export: +% +% void fzn_initialize(const char** args, size_t n_args) +% +% On Windows these functions use the Gecode blackbox calling convention. Native +% library blackboxes can be called concurrently by parallel search workers and +% must be thread-safe. +% +% blackbox_exec starts a persistent subprocess. For each call, Gecode writes one +% line to stdin: +% +% comma-separated integer inputs ; comma-separated float inputs +% +% The process must answer with one line in the same format for the expected +% integer and float outputs. Outputs must be finite and in Gecode's numeric +% ranges. Response lines are limited to 1 MiB. The helper is trusted code and +% must keep reading requests and writing complete newline-terminated responses; +% otherwise it can block the solver. During parallel search each worker thread +% gets its own process. +% +% All blackboxes must be deterministic from FlatZinc's point of view: the same +% input arrays must produce the same output arrays. Implementations may cache +% internally as long as the observable result does not depend on call order. annotation blackbox_dll(string: library); annotation blackbox_dll(string: library, array[int] of string: args); diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn index 44407aedf8..5706f7ad73 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn @@ -1,5 +1,8 @@ include "blackbox_annotations.mzn"; +% Value blackbox. The output arrays are constrained to the values returned by +% the selected blackbox when all input variables are fixed. If both input arrays +% are empty, the blackbox is evaluated once at posting. predicate fzn_blackbox( array[int] of var int: int_input, array[int] of var float: float_input, diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn index 68fa9640d0..cc79ad49b3 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn @@ -1,5 +1,10 @@ include "blackbox_annotations.mzn"; +% Bounds blackbox. The integer and float variables are encoded as lower/upper +% bound pairs, and the returned pairs narrow those same variables. flat_reason +% is the MiniZinc explanation encoding used to select which variable bounds can +% reschedule the blackbox. If it selects no dependencies, the blackbox is +% evaluated once at posting. predicate fzn_blackbox_bounds( array[int] of var int: int_input, array[int] of var float: float_input, diff --git a/gecode/flatzinc/registry.cpp b/gecode/flatzinc/registry.cpp index 8655fae1e3..18fcafad8b 100755 --- a/gecode/flatzinc/registry.cpp +++ b/gecode/flatzinc/registry.cpp @@ -1668,29 +1668,52 @@ namespace Gecode { namespace FlatZinc { void blackbox_source(AST::Node* ann, std::string& mode, std::string& instantiation, std::vector& args) { + auto error = [](const std::string& message) { + throw FlatZinc::Error("Registry", message); + }; + auto string_arg = [&](AST::Node* n, const char* what) { + if ((n == nullptr) || !n->isString()) { + error(std::string("Malformed blackbox annotation: ") + what + + " must be a string."); + } + return n->getString(); + }; AST::Call* c = nullptr; - if (ann->hasCall("blackbox_dll")) { + bool has_dll = (ann != nullptr) && ann->hasCall("blackbox_dll"); + bool has_exec = (ann != nullptr) && ann->hasCall("blackbox_exec"); + if (has_dll && has_exec) { + error("Blackbox constraint has multiple execution method annotations."); + } else if (has_dll) { c = ann->getCall("blackbox_dll"); mode = "dll"; - } else if (ann->hasCall("blackbox_exec")) { + } else if (has_exec) { c = ann->getCall("blackbox_exec"); mode = "exec"; } else { - throw FlatZinc::Error("Registry", - "Blackbox constraint is missing a valid annotation specifying execution method."); + error("Blackbox constraint is missing a valid annotation specifying " + "execution method."); + } + if ((c == nullptr) || (c->args == nullptr)) { + error("Malformed blackbox annotation: missing target."); } // For a single-argument call `args` is the bare argument node; for the // `(target, args)` form it is an array of the two arguments. if (AST::Array* arr = dynamic_cast(c->args)) { - instantiation = arr->a[0]->getString(); - if (arr->a.size() > 1) { - AST::Array* al = arr->a[1]->getArray(); - for (unsigned int i = 0; i < al->a.size(); i++) { - args.push_back(al->a[i]->getString()); - } + if (arr->a.size() != 2) { + error("Malformed blackbox annotation: expected a target string and " + "an argument array."); + } + instantiation = string_arg(arr->a[0], "target"); + if (!arr->a[1]->isArray()) { + error("Malformed blackbox annotation: argument list must be an array " + "of strings."); + } + AST::Array* al = arr->a[1]->getArray(); + for (unsigned int i = 0; i < al->a.size(); i++) { + args.push_back(string_arg(al->a[i], "argument")); } } else { - instantiation = c->args->getString(); + instantiation = string_arg(c->args, "target"); } } diff --git a/test/flatzinc.cpp b/test/flatzinc.cpp index 1d21450ad1..0b62985122 100755 --- a/test/flatzinc.cpp +++ b/test/flatzinc.cpp @@ -70,6 +70,10 @@ namespace Test { namespace FlatZinc { : Base("FlatZinc::"+name), _name(name), _source(source), _expected(expected), _allSolutions(allSolutions), _cmdlineOpt(cmdlineOpt) {} + FlatZincErrorTest::FlatZincErrorTest(const std::string& name, + const std::string& source) + : FlatZincTest(name, source, "") {} + bool FlatZincTest::run(void) { using namespace Gecode; @@ -121,6 +125,42 @@ namespace Test { namespace FlatZinc { return true; } + bool + FlatZincErrorTest::run(void) { + using namespace Gecode; + Support::Timer t_total; + t_total.start(); + Gecode::FlatZinc::FlatZincOptions fznopt("Gecode/FlatZinc"); + Gecode::FlatZinc::Printer p; + Gecode::FlatZinc::FlatZincSpace* fg = nullptr; + try { + std::stringstream ss(_source); + fg = Gecode::FlatZinc::parse(ss, p, olog); + if (fg) { + fg->createBranchers(p, fg->solveAnnotations(), fznopt, + false, olog); + fg->shrinkArrays(p); + std::ostringstream os; + fg->run(os, p, fznopt, t_total); + } + delete fg; + return false; + } catch (Gecode::FlatZinc::Error& e) { + delete fg; + if (opt.log) + olog << ind(2) << "Expected FlatZinc error : " + << e.toString() << std::endl; + return true; + } catch (Gecode::Exception& e) { + delete fg; + if (opt.log) + olog << ind(2) << "Expected Gecode exception : " + << e.what() << std::endl; + return true; + } + return false; + } + }} // STATISTICS: test-flatzinc diff --git a/test/flatzinc.hh b/test/flatzinc.hh index 3a554825e2..fe4ee74eed 100644 --- a/test/flatzinc.hh +++ b/test/flatzinc.hh @@ -67,6 +67,14 @@ namespace Test { virtual bool run(void); }; + class FlatZincErrorTest : public FlatZincTest { + public: + /// Construct and register test + FlatZincErrorTest(const std::string& name, const std::string& source); + /// Perform test + virtual bool run(void); + }; + } } diff --git a/test/flatzinc/blackbox.cpp b/test/flatzinc/blackbox.cpp new file mode 100644 index 0000000000..badbfcd7d9 --- /dev/null +++ b/test/flatzinc/blackbox.cpp @@ -0,0 +1,198 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Copyright: + * Jip J. Dekker, 2026 + * + * This file is part of Gecode, the generic constraint + * development environment: + * http://www.gecode.dev + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "test/flatzinc.hh" + +#ifdef GECODE_HAS_FLOAT_VARS + +namespace Test { namespace FlatZinc { + + namespace { + const char* blackbox_decl = + "predicate gecode_blackbox(" + "array[int] of var int: int_input, " + "array[int] of var float: float_input, " + "array[int] of var int: int_output, " + "array[int] of var float: float_output);\n"; + + const char* blackbox_bounds_decl = + "predicate gecode_blackbox_bounds(" + "array[int] of var int: int_input, " + "array[int] of var float: float_input, " + "array[int] of int: flat_reason);\n"; + + const char* python_value_7 = + "blackbox_exec(\"python3\", " + "[\"-u\", \"-c\", " + "\"import sys; [print(chr(55)+chr(59), flush=True) " + "for line in sys.stdin]\"])"; + + const char* python_bounds_5 = + "blackbox_exec(\"python3\", " + "[\"-u\", \"-c\", " + "\"import sys; [print(chr(53)+chr(44)+chr(53)+chr(59), " + "flush=True) for line in sys.stdin]\"])"; + + const char* python_bounds_5_twice = + "blackbox_exec(\"python3\", " + "[\"-u\", \"-c\", " + "\"import sys; [print(chr(53)+chr(44)+chr(53)+chr(44)+chr(53)+" + "chr(44)+chr(53)+chr(59), flush=True) for line in sys.stdin]\"])"; + + const char* python_mixed_bounds_5 = + "blackbox_exec(\"python3\", " + "[\"-u\", \"-c\", " + "\"import sys; [print(chr(53)+chr(44)+chr(53)+chr(59)+chr(53)+" + "chr(44)+chr(53), flush=True) for line in sys.stdin]\"])"; + + const char* python_nan = + "blackbox_exec(\"python3\", " + "[\"-u\", \"-c\", " + "\"import sys; [print(chr(59)+chr(110)+chr(97)+chr(110), " + "flush=True) for line in sys.stdin]\"])"; + + const char* python_nul = + "blackbox_exec(\"python3\", " + "[\"-u\", \"-c\", " + "\"import sys; [print(chr(59)+chr(0)+chr(120), " + "flush=True) for line in sys.stdin]\"])"; + } + + namespace Blackbox { + class Create { + public: + /// Perform creation and registration + Create(void) { + (void) new FlatZincTest("blackbox::constant_value", + std::string(blackbox_decl) + + "var 7..7: y :: output_var;\n" + "constraint gecode_blackbox([], [], [y], []) :: " + + python_value_7 + + ";\n" + "solve satisfy;\n", + "y = 7;\n----------\n"); + + (void) new FlatZincTest("blackbox::constant_value_unsat", + std::string(blackbox_decl) + + "var 8..8: y;\n" + "constraint gecode_blackbox([], [], [y], []) :: " + + python_value_7 + + ";\n" + "solve satisfy;\n", + "=====UNSATISFIABLE=====\n"); + + (void) new FlatZincTest("blackbox::reason_independent_bounds", + std::string(blackbox_bounds_decl) + + "var 5..5: x :: output_var;\n" + "constraint gecode_blackbox_bounds([x], [], [1,0,0]) :: " + + python_bounds_5 + + ";\n" + "solve satisfy;\n", + "x = 5;\n----------\n"); + + (void) new FlatZincTest("blackbox::reason_independent_bounds_unsat", + std::string(blackbox_bounds_decl) + + "var 6..6: x;\n" + "constraint gecode_blackbox_bounds([x], [], [1,0,0]) :: " + + python_bounds_5 + + ";\n" + "solve satisfy;\n", + "=====UNSATISFIABLE=====\n"); + + (void) new FlatZincTest("blackbox::reason_dependent_bounds", + std::string(blackbox_bounds_decl) + + "var 5..5: x :: output_var;\n" + "constraint gecode_blackbox_bounds([x], [], [1,1,1,1,0]) :: " + + python_bounds_5 + + ";\n" + "solve satisfy;\n", + "x = 5;\n----------\n"); + + (void) new FlatZincErrorTest("blackbox::missing_bounds_reason_entry", + std::string(blackbox_bounds_decl) + + "var 0..10: x;\n" + "var 0.0..10.0: y;\n" + "constraint gecode_blackbox_bounds([x], [y], [1,0,0]) :: " + + python_mixed_bounds_5 + + ";\n" + "solve satisfy;\n"); + + (void) new FlatZincErrorTest("blackbox::duplicate_bounds_reason_entry", + std::string(blackbox_bounds_decl) + + "var 0..10: x;\n" + "var 0..10: y;\n" + "constraint gecode_blackbox_bounds([x,y], [], [1,0,0,1,0,0]) :: " + + python_bounds_5_twice + + ";\n" + "solve satisfy;\n"); + + (void) new FlatZincErrorTest("blackbox::invalid_bounds_reason_code", + std::string(blackbox_bounds_decl) + + "var 0..10: x;\n" + "constraint gecode_blackbox_bounds([x], [], [1,1,1,0,0]) :: " + + python_bounds_5 + + ";\n" + "solve satisfy;\n"); + + (void) new FlatZincErrorTest("blackbox::invalid_float_output", + std::string(blackbox_decl) + + "var 0.0..10.0: y;\n" + "constraint gecode_blackbox([], [], [], [y]) :: " + + python_nan + + ";\n" + "solve satisfy;\n"); + + (void) new FlatZincErrorTest("blackbox::nul_output", + std::string(blackbox_decl) + + "constraint gecode_blackbox([], [], [], []) :: " + + python_nul + + ";\n" + "solve satisfy;\n"); + + (void) new FlatZincErrorTest("blackbox::malformed_annotation", + std::string(blackbox_decl) + + "var 0..1: y;\n" + "constraint gecode_blackbox([], [], [y], []) :: " + "blackbox_exec([]);\n" + "solve satisfy;\n"); + } + }; + + Create c; + } + +}} + +#endif + +// STATISTICS: test-flatzinc From 0c0a9ae023bf241b1f2d9f7df0ed5192239ca5fc Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Fri, 10 Jul 2026 20:53:02 +0200 Subject: [PATCH 05/14] Share blackbox backends across search Reuse model-local backends while giving concurrent workers independent callback instances or persistent process sessions. --- gecode/flatzinc.hh | 3 + gecode/flatzinc/blackbox.cpp | 256 +++++++++++++++++++++++++++++++---- gecode/flatzinc/blackbox.hh | 92 ++++++++----- gecode/flatzinc/flatzinc.cpp | 170 +++++++++++++++-------- gecode/flatzinc/registry.cpp | 4 +- test/flatzinc.cpp | 15 +- test/flatzinc.hh | 3 +- test/flatzinc/blackbox.cpp | 36 +++++ 8 files changed, 457 insertions(+), 122 deletions(-) diff --git a/gecode/flatzinc.hh b/gecode/flatzinc.hh index 1c187d8ed8..4f38d5dd85 100755 --- a/gecode/flatzinc.hh +++ b/gecode/flatzinc.hh @@ -604,6 +604,9 @@ namespace Gecode { namespace FlatZinc { /// Post a constraint specified by \a ce void postConstraints(std::vector& ces); + /// Return opaque state used while posting blackbox constraints + SharedHandle& blackBoxState(void); + /// Post the solve item void solve(AST::Array* annotation); /// Post that integer variable \a var should be minimized diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp index 427c06aa0f..19dc561a6d 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox.cpp @@ -36,11 +36,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -56,6 +58,7 @@ #else #include #include +#include #include #include #include @@ -265,9 +268,9 @@ const size_t max_exec_response_size = 1024 * 1024; } // namespace -BlackBoxLibrary::BlackBoxLibrary(const std::string &name, - const std::vector &args) - : library(nullptr), library_fzn_blackbox(nullptr) { +BlackBoxLibrary::BlackBoxLibrary(const std::string &name) + : library(nullptr), library_fzn_blackbox(nullptr), + library_fzn_initialize(nullptr) { std::string loadError; void *loaded = nullptr; #ifdef _WIN32 @@ -341,39 +344,47 @@ BlackBoxLibrary::BlackBoxLibrary(const std::string &name, symError); } - // Optionally call the initialisation function with the given arguments. It is - // not an error for the library to omit `fzn_initialize`. - void(GECODE_BLACKBOX_CALL *dll_fzn_initialize)(const char **, size_t) = - nullptr; #ifdef _WIN32 - dll_fzn_initialize = reinterpret_cast( + char path[MAX_PATH]; + DWORD path_size = GetModuleFileNameA(static_cast(loaded), path, + static_cast(sizeof(path))); + if ((path_size > 0) && (path_size < sizeof(path))) { + library_identity.assign(path, path_size); + } +#else + Dl_info info; + if ((dladdr(reinterpret_cast(library_fzn_blackbox), &info) != 0) && + (info.dli_fname != nullptr)) { + char resolved[PATH_MAX]; + if (realpath(info.dli_fname, resolved) != nullptr) { + library_identity = resolved; + } else { + library_identity = info.dli_fname; + } + } +#endif + if (library_identity.empty()) { + library_identity = name; + } + + // Look up the optional initialization function. Calling it is deferred + // until the model-local backend cache has checked its configuration. +#ifdef _WIN32 + library_fzn_initialize = reinterpret_cast( GetProcAddress((HMODULE)loaded, "fzn_initialize")); #if defined(_M_IX86) || defined(__i386__) - if (!dll_fzn_initialize) { - dll_fzn_initialize = reinterpret_cast( + if (!library_fzn_initialize) { + library_fzn_initialize = reinterpret_cast( GetProcAddress((HMODULE)loaded, "_fzn_initialize@8")); } - if (!dll_fzn_initialize) { - dll_fzn_initialize = reinterpret_cast( + if (!library_fzn_initialize) { + library_fzn_initialize = reinterpret_cast( GetProcAddress((HMODULE)loaded, "fzn_initialize@8")); } #endif #else - *(void **)(&dll_fzn_initialize) = dlsym(loaded, "fzn_initialize"); + *(void **)(&library_fzn_initialize) = dlsym(loaded, "fzn_initialize"); #endif - try { - if (dll_fzn_initialize != nullptr) { - std::vector argv; - argv.reserve(args.size()); - for (const std::string &a : args) { - argv.push_back(a.c_str()); - } - dll_fzn_initialize(argv.data(), argv.size()); - } - } catch (...) { - close_library(loaded); - throw; - } library = loaded; } @@ -381,6 +392,23 @@ BlackBoxLibrary::~BlackBoxLibrary() { close_library(library); } +void +BlackBoxLibrary::initialize(const std::vector &args) { + if (library_fzn_initialize != nullptr) { + std::vector argv; + argv.reserve(args.size()); + for (const std::string &a : args) { + argv.push_back(a.c_str()); + } + library_fzn_initialize(argv.data(), argv.size()); + } +} + +const std::string & +BlackBoxLibrary::identity(void) const { + return library_identity; +} + void BlackBoxLibrary::run(const std::vector &int_in, const std::vector &float_in, @@ -988,6 +1016,158 @@ BlackBoxExec::~BlackBoxExec(void) { sessions.clear(); } +class BlackBoxState : public SharedHandle::Object { +protected: + class ExecEntry { + public: + std::string program; + std::vector args; + BlackBoxHandle handle; + ExecEntry(const std::string &program0, const std::vector &args0, + const BlackBoxHandle &handle0) + : program(program0), args(args0), handle(handle0) {} + }; + class LibraryEntry { + public: + std::string identity; + std::vector args; + std::vector names; + BlackBoxHandle handle; + LibraryEntry(const std::string &identity0, + const std::vector &args0, + const std::string &name, const BlackBoxHandle &handle0) + : identity(identity0), args(args0), names(1, name), handle(handle0) {} + }; + + mutable Support::Mutex mutex; + std::vector exec; + std::vector library; + std::exception_ptr exception; + std::atomic error_recorded; + +public: + BlackBoxState(void) : error_recorded(false) {} + BlackBoxHandle blackBox(const std::string &mode, + const std::string &instantiation, + const std::vector &args); + void fail(std::exception_ptr e); + bool failed(void) const; + void rethrow(void) const; +}; + +BlackBoxStateHandle +BlackBoxStateHandle::init(SharedHandle &handle) { + BlackBoxStateHandle state(handle); + if (!state) { + state.object(new BlackBoxState); + handle = state; + } + return state; +} + +BlackBoxHandle +BlackBoxStateHandle::blackBox(const std::string &mode, + const std::string &instantiation, + const std::vector &args) const { + return static_cast(object())->blackBox(mode, instantiation, + args); +} + +void +BlackBoxStateHandle::fail(std::exception_ptr e) const { + static_cast(object())->fail(e); +} + +bool +BlackBoxStateHandle::failed(void) const { + return static_cast(*this) && + static_cast(object())->failed(); +} + +void +BlackBoxStateHandle::rethrow(void) const { + if (*this) { + static_cast(object())->rethrow(); + } +} + +BlackBoxHandle +BlackBoxState::blackBox(const std::string &mode, + const std::string &instantiation, + const std::vector &args) { + Support::Lock lock(mutex); + if (mode == "exec") { + for (const ExecEntry &e : exec) { + if ((e.program == instantiation) && (e.args == args)) { + return e.handle; + } + } + BlackBoxHandle handle(new BlackBoxExec(instantiation, args)); + exec.push_back(ExecEntry(instantiation, args, handle)); + return handle; + } + if (mode == "dll") { + for (LibraryEntry &e : library) { + if (std::find(e.names.begin(), e.names.end(), instantiation) != + e.names.end()) { + if (e.args != args) { + throw Error("Blackbox", "Conflicting initialization arguments for " + "dynamic library `" + e.identity + "'"); + } + return e.handle; + } + } + + BlackBoxHandle handle(new BlackBoxLibrary(instantiation)); + BlackBoxLibrary *black_box = + static_cast(handle()); + for (LibraryEntry &e : library) { + if (e.identity == black_box->identity()) { + if (e.args != args) { + throw Error("Blackbox", "Conflicting initialization arguments for " + "dynamic library `" + e.identity + "'"); + } + e.names.push_back(instantiation); + return e.handle; + } + } + black_box->initialize(args); + library.push_back(LibraryEntry(black_box->identity(), args, instantiation, + handle)); + return handle; + } + throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); +} + +void +BlackBoxState::fail(std::exception_ptr e) { + Support::Lock lock(mutex); + if (!error_recorded.load(std::memory_order_relaxed)) { + exception = e; + error_recorded.store(true, std::memory_order_release); + } +} + +bool +BlackBoxState::failed(void) const { + return error_recorded.load(std::memory_order_acquire); +} + +void +BlackBoxState::rethrow(void) const { + if (!error_recorded.load(std::memory_order_acquire)) { + return; + } + std::exception_ptr e; + { + Support::Lock lock(mutex); + e = exception; + } + if (e != nullptr) { + std::rethrow_exception(e); + } +} + BlackBoxExec::Session &BlackBoxExec::session(void) { Support::Lock lock(mutex); for (Session *s : sessions) { @@ -1148,7 +1328,12 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { } #endif - black_box()->run(int_in, float_in, int_out, float_out); + try { + black_box()->run(int_in, float_in, int_out, float_out); + } catch (...) { + black_box_state.fail(std::current_exception()); + return ES_FAILED; + } for (int i = 0; i < int_out.size(); i++) { // std::cerr << int_out[i] << " "; @@ -1184,7 +1369,12 @@ ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { } #endif - black_box()->run(int_in, float_in, int_out, float_out); + try { + black_box()->run(int_in, float_in, int_out, float_out); + } catch (...) { + black_box_state.fail(std::current_exception()); + return ES_FAILED; + } for (int i = 0; i < ivar.size(); i++) { // std::cerr << int_out[i*2] << ".." << int_out[i*2+1] << " "; @@ -1201,7 +1391,8 @@ ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { return ES_NOFIX; } -void blackbox(Home home, const IntVarArgs &int_in, const IntVarArgs &int_out, +void blackbox(Home home, SharedHandle &black_box_state, + const IntVarArgs &int_in, const IntVarArgs &int_out, #ifdef GECODE_HAS_FLOAT_VARS const FloatVarArgs &float_in, const FloatVarArgs &float_out, #endif @@ -1216,11 +1407,13 @@ void blackbox(Home home, const IntVarArgs &int_in, const IntVarArgs &int_out, if (home.failed()) return; + BlackBoxStateHandle state = BlackBoxStateHandle::init(black_box_state); PostInfo pi(home); ExecStatus es = BlackBox::post(home, int_input, int_output, #ifdef GECODE_HAS_FLOAT_VARS float_input, float_output, #endif + state, mode, instantiation, args); GECODE_ES_FAIL(es); } @@ -1307,7 +1500,8 @@ static void reason_subscriptions(const std::vector &reason, int n_int, } } -void blackbox_bounds(Home home, const IntVarArgs &ivar, +void blackbox_bounds(Home home, SharedHandle &black_box_state, + const IntVarArgs &ivar, #ifdef GECODE_HAS_FLOAT_VARS const FloatVarArgs &fvar, #endif @@ -1331,6 +1525,7 @@ void blackbox_bounds(Home home, const IntVarArgs &ivar, if (home.failed()) return; + BlackBoxStateHandle state = BlackBoxStateHandle::init(black_box_state); PostInfo pi(home); ExecStatus es = BlackBoxBounds::post(home, int_var, #ifdef GECODE_HAS_FLOAT_VARS @@ -1340,6 +1535,7 @@ void blackbox_bounds(Home home, const IntVarArgs &ivar, #ifdef GECODE_HAS_FLOAT_VARS sub_float, #endif + state, mode, instantiation, args); GECODE_ES_FAIL(es); } diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index 14b8b2163c..b85449eada 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -77,9 +78,12 @@ public: /// search workers and must therefore be thread-safe. class BlackBoxLibrary : public BlackBoxFn { public: - BlackBoxLibrary(const std::string &name, - const std::vector &args); + BlackBoxLibrary(const std::string &name); ~BlackBoxLibrary(); + /// Initialize the library with the model-specific configuration + void initialize(const std::vector &args); + /// Return the loaded library identity + const std::string &identity(void) const; void run(const std::vector &int_in, const std::vector &float_in, std::vector &int_out, @@ -87,9 +91,11 @@ public: protected: void *library; + std::string library_identity; void(GECODE_BLACKBOX_CALL *library_fzn_blackbox)( const int64_t *, size_t, const double *, size_t, int64_t *, size_t, double *, size_t); + void(GECODE_BLACKBOX_CALL *library_fzn_initialize)(const char **, size_t); }; /// Implementation of a blackbox function that starts a separate process to @@ -131,6 +137,31 @@ public: BlackBoxFn *operator()() { return static_cast(object()); }; }; +/// Typed handle retained by blackbox propagators and search support. +class BlackBoxStateHandle : public SharedHandle { +public: + BlackBoxStateHandle(void) : SharedHandle() {} + BlackBoxStateHandle(const SharedHandle &handle) : SharedHandle(handle) {} + BlackBoxStateHandle(const BlackBoxStateHandle &handle) + : SharedHandle(handle) {} + BlackBoxStateHandle &operator=(const BlackBoxStateHandle &handle) { + return static_cast(SharedHandle::operator=(handle)); + } + + /// Initialize the model-local state held by \a handle, if necessary + static BlackBoxStateHandle init(SharedHandle &handle); + /// Return a cached blackbox backend, creating it if necessary + BlackBoxHandle blackBox(const std::string &mode, + const std::string &instantiation, + const std::vector &args) const; + /// Record the first exception raised while propagating a blackbox + void fail(std::exception_ptr e) const; + /// Whether a blackbox propagator has failed with an exception + bool failed(void) const; + /// Rethrow the first exception raised by a propagating blackbox + void rethrow(void) const; +}; + class BlackBox : public Propagator { protected: /// Integer variables considered as the integer input to the blackbox function @@ -152,10 +183,13 @@ protected: /// The handle ensures that the function implementation can be shared between /// copies of the propagator. BlackBoxHandle black_box; + /// State shared by all blackbox propagators in the model + BlackBoxStateHandle black_box_state; /// Constructor for cloning \a p BlackBox(Space &home, BlackBox &p) - : Propagator(home, p), black_box(p.black_box) { + : Propagator(home, p), black_box(p.black_box), + black_box_state(p.black_box_state) { int_input.update(home, p.int_input); int_output.update(home, p.int_output); #ifdef GECODE_HAS_FLOAT_VARS @@ -172,12 +206,13 @@ public: ViewArray &float_in, ViewArray &float_out, #endif - const BlackBoxHandle &black_box0) + const BlackBoxHandle &black_box0, + const BlackBoxStateHandle &black_box_state0) : Propagator(home), int_input(int_in), int_output(int_out), #ifdef GECODE_HAS_FLOAT_VARS float_input(float_in), float_output(float_out), #endif - black_box(black_box0) { + black_box(black_box0), black_box_state(black_box_state0) { int_input.subscribe(home, *this, Int::PC_INT_VAL); #ifdef GECODE_HAS_FLOAT_VARS float_input.subscribe(home, *this, Float::PC_FLOAT_VAL); @@ -207,6 +242,7 @@ public: #endif home.ignore(*this, AP_DISPOSE); black_box.~BlackBoxHandle(); + black_box_state.~BlackBoxStateHandle(); (void)Propagator::dispose(home); return sizeof(*this); }; @@ -223,19 +259,12 @@ public: ViewArray &float_input, ViewArray &float_output, #endif + const BlackBoxStateHandle &black_box_state, const std::string &mode, const std::string &instantiation, const std::vector &args) { - BlackBoxFn *black_box(nullptr); - if (mode == "dll") { - black_box = new BlackBoxLibrary(instantiation, args); - } else if (mode == "exec") { - black_box = new BlackBoxExec(instantiation, args); - } else { - throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); - } - - BlackBoxHandle black_box_handle(black_box); + BlackBoxHandle black_box_handle = + black_box_state.blackBox(mode, instantiation, args); if ((int_input.size() == 0) #ifdef GECODE_HAS_FLOAT_VARS && (float_input.size() == 0) @@ -268,7 +297,7 @@ public: #ifdef GECODE_HAS_FLOAT_VARS float_input, float_output, #endif - black_box_handle); + black_box_handle, black_box_state); return ES_OK; } }; @@ -302,6 +331,8 @@ protected: /// The handle ensures that the function implementation can be shared between /// copies of the propagator. BlackBoxHandle black_box; + /// State shared by all blackbox propagators in the model + BlackBoxStateHandle black_box_state; /// Constructor for cloning \a p BlackBoxBounds(Space &home, BlackBoxBounds &p) @@ -309,7 +340,7 @@ protected: #ifdef GECODE_HAS_FLOAT_VARS sub_float(p.sub_float), #endif - black_box(p.black_box) { + black_box(p.black_box), black_box_state(p.black_box_state) { ivar.update(home, p.ivar); #ifdef GECODE_HAS_FLOAT_VARS fvar.update(home, p.fvar); @@ -326,7 +357,8 @@ public: #ifdef GECODE_HAS_FLOAT_VARS SharedArray sub_float0, #endif - const BlackBoxHandle &black_box0) + const BlackBoxHandle &black_box0, + const BlackBoxStateHandle &black_box_state0) : Propagator(home), ivar(ivar), #ifdef GECODE_HAS_FLOAT_VARS fvar(fvar), @@ -335,7 +367,7 @@ public: #ifdef GECODE_HAS_FLOAT_VARS sub_float(sub_float0), #endif - black_box(black_box0) { + black_box(black_box0), black_box_state(black_box_state0) { for (int i = 0; i < ivar.size(); i++) { if (sub_int[i]) { ivar[i].subscribe(home, *this, Int::PC_INT_BND); @@ -391,6 +423,7 @@ public: home.ignore(*this, AP_DISPOSE); home.ignore(*this, AP_WEAKLY); black_box.~BlackBoxHandle(); + black_box_state.~BlackBoxStateHandle(); sub_int.~SharedArray(); #ifdef GECODE_HAS_FLOAT_VARS sub_float.~SharedArray(); @@ -413,19 +446,12 @@ public: #ifdef GECODE_HAS_FLOAT_VARS SharedArray sub_float, #endif + const BlackBoxStateHandle &black_box_state, const std::string &mode, const std::string &instantiation, const std::vector &args) { - BlackBoxFn *black_box(nullptr); - if (mode == "dll") { - black_box = new BlackBoxLibrary(instantiation, args); - } else if (mode == "exec") { - black_box = new BlackBoxExec(instantiation, args); - } else { - throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); - } - - BlackBoxHandle black_box_handle(black_box); + BlackBoxHandle black_box_handle = + black_box_state.blackBox(mode, instantiation, args); bool has_subscription = false; for (int i = 0; i < ivar.size(); i++) { has_subscription = has_subscription || sub_int[i]; @@ -478,19 +504,21 @@ public: #ifdef GECODE_HAS_FLOAT_VARS sub_float, #endif - black_box_handle); + black_box_handle, black_box_state); return ES_OK; } }; -void blackbox(Home home, const IntVarArgs &int_in, const IntVarArgs &int_out, +void blackbox(Home home, SharedHandle &black_box_state, + const IntVarArgs &int_in, const IntVarArgs &int_out, #ifdef GECODE_HAS_FLOAT_VARS const FloatVarArgs &float_in, const FloatVarArgs &float_out, #endif const std::string &mode, const std::string &instantiation, const std::vector &args); -void blackbox_bounds(Home home, const IntVarArgs &ivar, +void blackbox_bounds(Home home, SharedHandle &black_box_state, + const IntVarArgs &ivar, #ifdef GECODE_HAS_FLOAT_VARS const FloatVarArgs &fvar, #endif diff --git a/gecode/flatzinc/flatzinc.cpp b/gecode/flatzinc/flatzinc.cpp index c3161602ba..090cfebc59 100644 --- a/gecode/flatzinc/flatzinc.cpp +++ b/gecode/flatzinc/flatzinc.cpp @@ -38,6 +38,7 @@ */ #include +#include #include #include #include @@ -769,6 +770,9 @@ namespace Gecode { namespace FlatZinc { /// Hash table of DFAs DFASet dfaSet; + /// Opaque state shared by blackbox propagators in this model + SharedHandle blackBoxState; + /// Initialize FlatZincSpaceInitData(void) {} }; @@ -864,6 +868,12 @@ namespace Gecode { namespace FlatZinc { branchInfo.init(); } + SharedHandle& + FlatZincSpace::blackBoxState(void) { + assert(_initData != nullptr); + return _initData->blackBoxState; + } + void FlatZincSpace::init(int intVars, int boolVars, int setVars, int floatVars) { @@ -1739,6 +1749,23 @@ namespace Gecode { namespace FlatZinc { #endif + class FlatZincStop : public Search::Stop { + protected: + Search::Stop* stop_object; + BlackBoxStateHandle black_box_state; + public: + FlatZincStop(Search::Stop* stop_object0, + const BlackBoxStateHandle& black_box_state0) + : stop_object(stop_object0), black_box_state(black_box_state0) {} + bool stop(const Search::Statistics& s, const Search::Options& o) override { + return black_box_state.failed() || + ((stop_object != nullptr) && stop_object->stop(s,o)); + } + ~FlatZincStop(void) { + delete stop_object; + } + }; + template class Engine> void FlatZincSpace::runEngine(std::ostream& out, const Printer& p, @@ -1836,7 +1863,9 @@ namespace Gecode { namespace FlatZinc { if (opt.mode() == SM_GIST) { FZPrintingInspector pi(p); FZPrintingComparator pc(p); + BlackBoxStateHandle black_box_state(blackBoxState()); (void) GistEngine >::explore(this,opt,&pi,&pc); + black_box_state.rethrow(); return; } #endif @@ -1847,9 +1876,14 @@ namespace Gecode { namespace FlatZinc { if (status(sstat) != SS_FAILED) { n_p = PropagatorGroup::all.size(*this); } + BlackBoxStateHandle black_box_state(blackBoxState()); + black_box_state.rethrow(); Search::Options o; - o.stop = Driver::CombinedStop::create(opt.node(), opt.fail(), opt.time(), opt.restart_limit(), - true); + o.stop = Driver::CombinedStop::create(opt.node(), opt.fail(), opt.time(), + opt.restart_limit(), true); + if (black_box_state) { + o.stop = new FlatZincStop(o.stop, black_box_state); + } o.c_d = opt.c_d(); o.a_d = opt.a_d(); @@ -1873,68 +1907,94 @@ namespace Gecode { namespace FlatZinc { o.cutoff = new Search::CutoffAppend(new Search::CutoffConstant(0), 1, Driver::createCutoff(opt)); if (opt.interrupt()) Driver::CombinedStop::installCtrlHandler(true); - { - Meta se(this,o); - int noOfSolutions = opt.solutions(); - if (noOfSolutions == -1) { - noOfSolutions = (_method == SAT) ? 1 : 0; - } - bool printAll = _method == SAT || opt.allSolutions() || noOfSolutions != 0; - int findSol = noOfSolutions; - FlatZincSpace* sol = nullptr; - while (FlatZincSpace* next_sol = se.next()) { - delete sol; - sol = next_sol; - if (printAll) { - sol->print(out, p); - out << "----------" << std::endl; + int noOfSolutions = opt.solutions(); + if (noOfSolutions == -1) { + noOfSolutions = (_method == SAT) ? 1 : 0; + } + bool printAll = _method == SAT || opt.allSolutions() || noOfSolutions != 0; + int findSol = noOfSolutions; + bool solution_limit_reached = false; + bool engine_stopped = false; + Gecode::Search::Statistics stat; + FlatZincSpace* sol = nullptr; + try { + { + Meta se(this,o); + while (FlatZincSpace* next_sol = se.next()) { + if (black_box_state.failed()) { + delete next_sol; + break; + } + delete sol; + sol = next_sol; + if (printAll) { + sol->print(out, p); + out << "----------" << std::endl; + } + if (--findSol == 0) { + solution_limit_reached = true; + break; + } } - if (--findSol==0) - goto stopped; - } - if (sol && !printAll) { - sol->print(out, p); - out << "----------" << std::endl; - } - if (!se.stopped()) { - if (sol) { - out << "==========" << std::endl; - } else { - out << "=====UNSATISFIABLE=====" << std::endl; + engine_stopped = se.stopped(); + if (opt.mode() == SM_STAT) { + stat = se.statistics(); } - } else if (!sol) { - out << "=====UNKNOWN=====" << std::endl; } + } catch (...) { delete sol; - stopped: if (opt.interrupt()) Driver::CombinedStop::installCtrlHandler(false); - if (opt.mode() == SM_STAT) { - Gecode::Search::Statistics stat = se.statistics(); - double totalTime = (t_total.stop() / 1000.0); - double solveTime = (t_solve.stop() / 1000.0); - double initTime = totalTime - solveTime; - out << std::endl - << "%%%mzn-stat: initTime=" << initTime - << std::endl; - out << "%%%mzn-stat: solveTime=" << solveTime - << std::endl; - out << "%%%mzn-stat: solutions=" - << std::abs(noOfSolutions - findSol) << std::endl - << "%%%mzn-stat: variables=" - << (intVarCount + boolVarCount + setVarCount) << std::endl - << "%%%mzn-stat: propagators=" << n_p << std::endl - << "%%%mzn-stat: propagations=" << sstat.propagate+stat.propagate << std::endl - << "%%%mzn-stat: nodes=" << stat.node << std::endl - << "%%%mzn-stat: failures=" << stat.fail << std::endl - << "%%%mzn-stat: restarts=" << stat.restart << std::endl - << "%%%mzn-stat: peakDepth=" << stat.depth << std::endl - << "%%%mzn-stat-end" << std::endl - << std::endl; - } + delete o.stop; + delete o.tracer; + throw; } + if (opt.interrupt()) + Driver::CombinedStop::installCtrlHandler(false); delete o.stop; delete o.tracer; + if (black_box_state.failed()) { + delete sol; + black_box_state.rethrow(); + } + if (sol && !printAll) { + sol->print(out, p); + out << "----------" << std::endl; + } + if (!solution_limit_reached) { + if (!engine_stopped) { + if (sol) { + out << "==========" << std::endl; + } else { + out << "=====UNSATISFIABLE=====" << std::endl; + } + } else if (!sol) { + out << "=====UNKNOWN=====" << std::endl; + } + } + delete sol; + if (opt.mode() == SM_STAT) { + double totalTime = (t_total.stop() / 1000.0); + double solveTime = (t_solve.stop() / 1000.0); + double initTime = totalTime - solveTime; + out << std::endl + << "%%%mzn-stat: initTime=" << initTime + << std::endl; + out << "%%%mzn-stat: solveTime=" << solveTime + << std::endl; + out << "%%%mzn-stat: solutions=" + << std::abs(noOfSolutions - findSol) << std::endl + << "%%%mzn-stat: variables=" + << (intVarCount + boolVarCount + setVarCount) << std::endl + << "%%%mzn-stat: propagators=" << n_p << std::endl + << "%%%mzn-stat: propagations=" << sstat.propagate+stat.propagate << std::endl + << "%%%mzn-stat: nodes=" << stat.node << std::endl + << "%%%mzn-stat: failures=" << stat.fail << std::endl + << "%%%mzn-stat: restarts=" << stat.restart << std::endl + << "%%%mzn-stat: peakDepth=" << stat.depth << std::endl + << "%%%mzn-stat-end" << std::endl + << std::endl; + } } #ifdef GECODE_HAS_QT diff --git a/gecode/flatzinc/registry.cpp b/gecode/flatzinc/registry.cpp index 18fcafad8b..cc83ec1e74 100755 --- a/gecode/flatzinc/registry.cpp +++ b/gecode/flatzinc/registry.cpp @@ -1733,7 +1733,7 @@ namespace Gecode { namespace FlatZinc { "Blackbox propagator cannot use floating point values when Gecode is compiled without floating point decision variable support."); } #endif - FlatZinc::blackbox(s, int_input, int_output, + FlatZinc::blackbox(s, s.blackBoxState(), int_input, int_output, #ifdef GECODE_HAS_FLOAT_VARS float_input, float_output, #endif @@ -1759,7 +1759,7 @@ float_input, float_output, for (int i = 0; i < flat_reason.size(); i++) { reason[i] = flat_reason[i]; } - FlatZinc::blackbox_bounds(s, ivar, + FlatZinc::blackbox_bounds(s, s.blackBoxState(), ivar, #ifdef GECODE_HAS_FLOAT_VARS fvar, #endif diff --git a/test/flatzinc.cpp b/test/flatzinc.cpp index 0b62985122..34a7cc48f3 100755 --- a/test/flatzinc.cpp +++ b/test/flatzinc.cpp @@ -71,8 +71,9 @@ namespace Test { namespace FlatZinc { _allSolutions(allSolutions), _cmdlineOpt(cmdlineOpt) {} FlatZincErrorTest::FlatZincErrorTest(const std::string& name, - const std::string& source) - : FlatZincTest(name, source, "") {} + const std::string& source, + std::vector cmdlineOpt) + : FlatZincTest(name, source, "", false, cmdlineOpt) {} bool FlatZincTest::run(void) { @@ -131,6 +132,16 @@ namespace Test { namespace FlatZinc { Support::Timer t_total; t_total.start(); Gecode::FlatZinc::FlatZincOptions fznopt("Gecode/FlatZinc"); + if (!_cmdlineOpt.empty()) { + std::string cmd("fzn-gecode"); + int argc = static_cast(_cmdlineOpt.size()) + 1; + std::vector argv(argc); + argv[0] = const_cast(cmd.data()); + for (int i = 1; i < argc; ++i) { + argv[i] = const_cast(_cmdlineOpt[i-1].data()); + } + fznopt.parse(argc, argv.data()); + } Gecode::FlatZinc::Printer p; Gecode::FlatZinc::FlatZincSpace* fg = nullptr; try { diff --git a/test/flatzinc.hh b/test/flatzinc.hh index fe4ee74eed..db51a1b4e3 100644 --- a/test/flatzinc.hh +++ b/test/flatzinc.hh @@ -70,7 +70,8 @@ namespace Test { class FlatZincErrorTest : public FlatZincTest { public: /// Construct and register test - FlatZincErrorTest(const std::string& name, const std::string& source); + FlatZincErrorTest(const std::string& name, const std::string& source, + std::vector cmdlineOpt = {}); /// Perform test virtual bool run(void); }; diff --git a/test/flatzinc/blackbox.cpp b/test/flatzinc/blackbox.cpp index badbfcd7d9..52a85414c2 100644 --- a/test/flatzinc/blackbox.cpp +++ b/test/flatzinc/blackbox.cpp @@ -86,6 +86,12 @@ namespace Test { namespace FlatZinc { "[\"-u\", \"-c\", " "\"import sys; [print(chr(59)+chr(0)+chr(120), " "flush=True) for line in sys.stdin]\"])"; + + const char* python_malformed = + "blackbox_exec(\"python3\", " + "[\"-u\", \"-c\", " + "\"import sys; [print(chr(120)+chr(59), " + "flush=True) for line in sys.stdin]\"])"; } namespace Blackbox { @@ -185,6 +191,36 @@ namespace Test { namespace FlatZinc { "constraint gecode_blackbox([], [], [y], []) :: " "blackbox_exec([]);\n" "solve satisfy;\n"); + + (void) new FlatZincErrorTest("blackbox::missing_exec_parallel", + std::string(blackbox_decl) + + "var 0..1: x :: output_var;\n" + "var 0..1: y :: output_var;\n" + "constraint gecode_blackbox([x], [], [y], []) :: " + "blackbox_exec(\"gecode-blackbox-missing-program\");\n" + "solve :: int_search([x], first_fail, indomain_min, complete) " + "satisfy;\n", + {"-p", "2"}); + + (void) new FlatZincErrorTest("blackbox::missing_exec_root_status", + std::string(blackbox_decl) + + "var 0..0: x :: output_var;\n" + "var 0..1: y :: output_var;\n" + "constraint gecode_blackbox([x], [], [y], []) :: " + "blackbox_exec(\"gecode-blackbox-missing-program\");\n" + "solve satisfy;\n", + {"-p", "2"}); + + (void) new FlatZincErrorTest("blackbox::malformed_exec_parallel", + std::string(blackbox_decl) + + "var 0..1: x :: output_var;\n" + "var 0..1: y :: output_var;\n" + "constraint gecode_blackbox([x], [], [y], []) :: " + + python_malformed + + ";\n" + "solve :: int_search([x], first_fail, indomain_min, complete) " + "satisfy;\n", + {"-p", "2"}); } }; From c7b0ea3ec6c4d83e3583c26e68175284219118f3 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Fri, 10 Jul 2026 20:53:02 +0200 Subject: [PATCH 06/14] Harden POSIX blackbox processes Use controlled posix_spawn setup, explicit descriptor inheritance, process groups, and bounded teardown. --- CMakeLists.txt | 26 +++ configure | 98 ++++++++++++ configure.ac | 43 +++++ gecode/flatzinc/blackbox.cpp | 296 +++++++++++++++++++++++------------ gecode/support/config.hpp.in | 9 ++ 5 files changed, 376 insertions(+), 96 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 66591a5881..a9b62b54b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -736,6 +736,32 @@ if(HAVE_FORCE_INLINE) set(forceinline "__forceinline") endif() +include(CheckCXXSourceCompiles) +check_cxx_source_compiles("#include +#ifndef POSIX_SPAWN_CLOEXEC_DEFAULT +#error POSIX_SPAWN_CLOEXEC_DEFAULT is unavailable +#endif +int main() { + posix_spawn_file_actions_t actions; + return posix_spawn_file_actions_addinherit_np(&actions, 2); +}" HAVE_POSIX_SPAWN_CLOEXEC_DEFAULT_AND_ADDINHERIT_NP) +if(HAVE_POSIX_SPAWN_CLOEXEC_DEFAULT_AND_ADDINHERIT_NP) + set(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT 1) + set(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP 1) +endif() + +check_cxx_source_compiles("#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include +int main() { + posix_spawn_file_actions_t actions; + return posix_spawn_file_actions_addclosefrom_np(&actions, 3); +}" HAVE_POSIX_SPAWN_ADDCLOSEFROM_NP) +if(HAVE_POSIX_SPAWN_ADDCLOSEFROM_NP) + set(GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP 1) +endif() + check_c_source_compiles("int main() { return __builtin_ffsll(0); }" HAVE_BUILTIN_FFSLL) if(HAVE_BUILTIN_FFSLL) set(GECODE_HAS_BUILTIN_FFSLL "/**/") diff --git a/configure b/configure index 084c071bce..7b1931d4a4 100755 --- a/configure +++ b/configure @@ -14458,6 +14458,104 @@ printf "%s\n" "#define HAVE_MMAP 1" >>confdefs.h fi rm -f conftest.mmap conftest.txt + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for POSIX_SPAWN_CLOEXEC_DEFAULT and posix_spawn_file_actions_addinherit_np" >&5 +printf %s "checking for POSIX_SPAWN_CLOEXEC_DEFAULT and posix_spawn_file_actions_addinherit_np... " >&6; } +if test ${ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#ifndef POSIX_SPAWN_CLOEXEC_DEFAULT +#error POSIX_SPAWN_CLOEXEC_DEFAULT is unavailable +#endif + +int +main (void) +{ + +posix_spawn_file_actions_t actions; +return posix_spawn_file_actions_addinherit_np(&actions, 2); + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO" +then : + ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np=yes +else case e in #( + e) ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np" >&5 +printf "%s\n" "$ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np" >&6; } +if test "x$ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np" = "xyes" +then : + + +printf "%s\n" "#define GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT 1" >>confdefs.h + + +printf "%s\n" "#define GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP 1" >>confdefs.h + + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for posix_spawn_file_actions_addclosefrom_np" >&5 +printf %s "checking for posix_spawn_file_actions_addclosefrom_np... " >&6; } +if test ${ac_cv_gecode_posix_spawn_addclosefrom_np+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include + +int +main (void) +{ + +posix_spawn_file_actions_t actions; +return posix_spawn_file_actions_addclosefrom_np(&actions, 3); + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO" +then : + ac_cv_gecode_posix_spawn_addclosefrom_np=yes +else case e in #( + e) ac_cv_gecode_posix_spawn_addclosefrom_np=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gecode_posix_spawn_addclosefrom_np" >&5 +printf "%s\n" "$ac_cv_gecode_posix_spawn_addclosefrom_np" >&6; } +if test "x$ac_cv_gecode_posix_spawn_addclosefrom_np" = "xyes" +then : + + +printf "%s\n" "#define GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP 1" >>confdefs.h + + +fi + GECODE_SAVE_LIBS=${LIBS} { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5 printf %s "checking for library containing dlopen... " >&6; } diff --git a/configure.ac b/configure.ac index 668cf8af68..14f1a145e8 100644 --- a/configure.ac +++ b/configure.ac @@ -279,6 +279,49 @@ AC_GECODE_CBS AC_GECODE_CPPROFILER AC_GECODE_FLEXBISON AC_FUNC_MMAP + +AC_CACHE_CHECK( + [for POSIX_SPAWN_CLOEXEC_DEFAULT and posix_spawn_file_actions_addinherit_np], + [ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np], + [AC_LINK_IFELSE( + [AC_LANG_PROGRAM([[ +#include +#ifndef POSIX_SPAWN_CLOEXEC_DEFAULT +#error POSIX_SPAWN_CLOEXEC_DEFAULT is unavailable +#endif +]], [[ +posix_spawn_file_actions_t actions; +return posix_spawn_file_actions_addinherit_np(&actions, 2); +]])], + [ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np=yes], + [ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np=no])]) +AS_IF([test "x$ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np" = "xyes"], [ + AC_DEFINE([GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT], [1], + [Whether POSIX_SPAWN_CLOEXEC_DEFAULT is available]) + AC_DEFINE([GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP], [1], + [Whether posix_spawn_file_actions_addinherit_np is available]) +]) + +AC_CACHE_CHECK( + [for posix_spawn_file_actions_addclosefrom_np], + [ac_cv_gecode_posix_spawn_addclosefrom_np], + [AC_LINK_IFELSE( + [AC_LANG_PROGRAM([[ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include +]], [[ +posix_spawn_file_actions_t actions; +return posix_spawn_file_actions_addclosefrom_np(&actions, 3); +]])], + [ac_cv_gecode_posix_spawn_addclosefrom_np=yes], + [ac_cv_gecode_posix_spawn_addclosefrom_np=no])]) +AS_IF([test "x$ac_cv_gecode_posix_spawn_addclosefrom_np" = "xyes"], [ + AC_DEFINE([GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP], [1], + [Whether posix_spawn_file_actions_addclosefrom_np is available]) +]) + GECODE_SAVE_LIBS=${LIBS} AC_SEARCH_LIBS([dlopen], [dl], [GECODE_DL_LIBS=${ac_cv_search_dlopen}], [GECODE_DL_LIBS=]) diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp index 19dc561a6d..0653050c2a 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox.cpp @@ -31,6 +31,10 @@ * */ +#if !defined(_WIN32) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE 1 +#endif + #include #include #include @@ -65,6 +69,7 @@ #include #include #include +#include #include extern char **environ; #endif @@ -163,6 +168,79 @@ move_from_standard_fd(int fd) { return nfd; } +class FileDescriptor { +private: + int fd; +public: + explicit FileDescriptor(int fd0=-1) : fd(fd0) {} + ~FileDescriptor(void) { reset(); } + + int get(void) const { return fd; } + int release(void) { + int fd0 = fd; + fd = -1; + return fd0; + } + void reset(int fd0=-1) { + if (fd != -1) { + ::close(fd); + } + fd = fd0; + } +}; + +int +move_away_from_standard_fd(FileDescriptor &fd) { + int old = fd.release(); + int nfd = move_from_standard_fd(old); + if (nfd == -1) { + fd.reset(old); + } else { + fd.reset(nfd); + } + return nfd; +} + +class SpawnFileActions { +private: + posix_spawn_file_actions_t actions; + bool initialized; +public: + SpawnFileActions(void) : initialized(false) {} + ~SpawnFileActions(void) { + if (initialized) { + posix_spawn_file_actions_destroy(&actions); + } + } + + int init(void) { + int err = posix_spawn_file_actions_init(&actions); + initialized = err == 0; + return err; + } + posix_spawn_file_actions_t *get(void) { return &actions; } +}; + +class SpawnAttributes { +private: + posix_spawnattr_t attr; + bool initialized; +public: + SpawnAttributes(void) : initialized(false) {} + ~SpawnAttributes(void) { + if (initialized) { + posix_spawnattr_destroy(&attr); + } + } + + int init(void) { + int err = posix_spawnattr_init(&attr); + initialized = err == 0; + return err; + } + posix_spawnattr_t *get(void) { return &attr; } +}; + int create_socketpair(int sv[2]) { #ifdef SOCK_CLOEXEC @@ -482,22 +560,38 @@ class BlackBoxExec::Session { const std::vector &args); void close_windows(void); #else - static bool reap_child(pid_t pid, int &status) { - pid_t r; + static void sleep_grace_period(void) { + struct timespec remaining = {0, 10000000}; + while ((nanosleep(&remaining, &remaining) == -1) && (errno == EINTR)) {} + } + + static bool child_exited(pid_t pid) { + siginfo_t info; do { - r = waitpid(pid, &status, WNOHANG); - } while ((r == -1) && (errno == EINTR)); - return (r == pid) || ((r == -1) && (errno == ECHILD)); + info.si_pid = 0; + if (waitid(P_PID, pid, &info, WEXITED | WNOHANG | WNOWAIT) == 0) { + return info.si_pid != 0; + } + } while (errno == EINTR); + return false; } - static bool wait_child(pid_t pid, int &status, int attempts) { + static void signal_group(pid_t pid, int signal) { + if ((kill(-pid, signal) == -1) && (errno == ESRCH)) { + return; + } + } + + static void wait_group(pid_t pid, int attempts) { for (int i = 0; i < attempts; i++) { - if (reap_child(pid, status)) { - return true; + if ((kill(-pid, 0) == -1) && (errno == ESRCH)) { + return; } - usleep(10000); + if (child_exited(pid)) { + return; + } + sleep_grace_period(); } - return reap_child(pid, status); } static void terminate_child(pid_t pid) { @@ -505,18 +599,10 @@ class BlackBoxExec::Session { return; } int status = 0; - if (wait_child(pid, status, 100)) { - return; - } - if (kill(-pid, SIGTERM) != 0) { - kill(pid, SIGTERM); - } - if (wait_child(pid, status, 100)) { - return; - } - if (kill(-pid, SIGKILL) != 0) { - kill(pid, SIGKILL); - } + // Keep the child unreaped until the group has received both signals. + signal_group(pid, SIGTERM); + wait_group(pid, 100); + signal_group(pid, SIGKILL); do { if (waitpid(pid, &status, 0) != -1) { return; @@ -524,6 +610,22 @@ class BlackBoxExec::Session { } while (errno == EINTR); } + static void check_sigchld(void) { + struct sigaction action; + if (sigaction(SIGCHLD, NULL, &action) != 0) { + throw Error("BlackBoxExec", last_error("SIGCHLD query failed")); + } + if ((action.sa_handler != SIG_DFL) +#ifdef SA_NOCLDWAIT + || (action.sa_flags & SA_NOCLDWAIT) +#endif + ) { + throw Error("BlackBoxExec", + "Cannot start a blackbox process unless SIGCHLD uses " + "SIG_DFL without SA_NOCLDWAIT"); + } + } + void open_posix(const std::string &program, const std::vector &args); void close_posix(void); @@ -633,10 +735,15 @@ class BlackBoxExec::Session { throw Error("BlackBoxExec", "Blackbox process provided an incomplete response"); } + int err = errno; + if (err == EINTR) { + clearerr(file_receive); + continue; + } throw Error("BlackBoxExec", std::string("Reading blackbox process output from pipe " "failed with errno ") + - std::to_string(errno)); + std::to_string(err)); } in_buffer += static_cast(ch); if (in_buffer.size() > max_exec_response_size) { @@ -859,132 +966,129 @@ BlackBoxExec::Session::open_posix(const std::string &program, const std::vector &args) { const int READ = 0; const int WRITE = 1; - int child_in[2] = {-1, -1}; - int child_out[2] = {-1, -1}; - if (create_socketpair(child_in) != 0) { + + std::vector argv; + argv.reserve(args.size() + 2); + argv.push_back(const_cast(program.c_str())); + for (const std::string &a : args) { + argv.push_back(const_cast(a.c_str())); + } + argv.push_back(nullptr); + +#if !((defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ + defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP)) || \ + defined(GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP)) + throw Error("BlackBoxExec", + "Persistent process blackboxes require a safe posix_spawn " + "descriptor-inheritance facility on this platform"); +#endif + check_sigchld(); + + FileDescriptor child_in[2]; + FileDescriptor child_out[2]; + int fds[2]; + if (create_socketpair(fds) != 0) { throw Error("BlackBoxExec", last_error("stdin socket creation failed")); } - if (create_socketpair(child_out) != 0) { - ::close(child_in[READ]); - ::close(child_in[WRITE]); + child_in[READ].reset(fds[READ]); + child_in[WRITE].reset(fds[WRITE]); + if (create_socketpair(fds) != 0) { throw Error("BlackBoxExec", last_error("stdout socket creation failed")); } - int fds[4] = {child_in[READ], child_in[WRITE], - child_out[READ], child_out[WRITE]}; - for (int i = 0; i < 4; i++) { - int moved = move_from_standard_fd(fds[i]); - if (moved == -1) { - int e = errno; - for (int j = 0; j < 4; j++) - ::close(fds[j]); - errno = e; + child_out[READ].reset(fds[READ]); + child_out[WRITE].reset(fds[WRITE]); + FileDescriptor *session_fds[] = { + &child_in[READ], &child_in[WRITE], + &child_out[READ], &child_out[WRITE] + }; + for (FileDescriptor *fd : session_fds) { + if (move_away_from_standard_fd(*fd) == -1) { throw Error("BlackBoxExec", last_error("moving session descriptors away from stdio " "failed")); } - fds[i] = moved; } - child_in[READ] = fds[0]; - child_in[WRITE] = fds[1]; - child_out[READ] = fds[2]; - child_out[WRITE] = fds[3]; - std::vector argv; - argv.reserve(args.size() + 2); - argv.push_back(const_cast(program.c_str())); - for (const std::string &a : args) { - argv.push_back(const_cast(a.c_str())); - } - argv.push_back(nullptr); - - posix_spawn_file_actions_t actions; - int err = posix_spawn_file_actions_init(&actions); + SpawnFileActions actions; + int err = actions.init(); if (err != 0) { - ::close(child_in[READ]); - ::close(child_in[WRITE]); - ::close(child_out[READ]); - ::close(child_out[WRITE]); errno = err; throw Error("BlackBoxExec", last_error("spawn file action init failed")); } - posix_spawnattr_t attr; - err = posix_spawnattr_init(&attr); + SpawnAttributes attr; + err = attr.init(); if (err != 0) { - posix_spawn_file_actions_destroy(&actions); - ::close(child_in[READ]); - ::close(child_in[WRITE]); - ::close(child_out[READ]); - ::close(child_out[WRITE]); errno = err; throw Error("BlackBoxExec", last_error("spawn attribute init failed")); } - err = posix_spawnattr_setpgroup(&attr, 0); + err = posix_spawnattr_setpgroup(attr.get(), 0); if (err == 0) { - err = posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETPGROUP); + short flags = POSIX_SPAWN_SETPGROUP; +#if defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ + defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP) + flags |= POSIX_SPAWN_CLOEXEC_DEFAULT; +#endif + err = posix_spawnattr_setflags(attr.get(), flags); } if (err == 0) { - err = posix_spawn_file_actions_adddup2(&actions, child_in[READ], + err = posix_spawn_file_actions_adddup2(actions.get(), + child_in[READ].get(), STDIN_FILENO); } if (err == 0) { - err = posix_spawn_file_actions_adddup2(&actions, child_out[WRITE], + err = posix_spawn_file_actions_adddup2(actions.get(), + child_out[WRITE].get(), STDOUT_FILENO); } +#if defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ + defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP) if (err == 0) { - err = posix_spawn_file_actions_addclose(&actions, child_in[READ]); - } - if (err == 0) { - err = posix_spawn_file_actions_addclose(&actions, child_in[WRITE]); + err = posix_spawn_file_actions_addinherit_np(actions.get(), + STDERR_FILENO); } +#elif defined(GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP) if (err == 0) { - err = posix_spawn_file_actions_addclose(&actions, child_out[READ]); - } - if (err == 0) { - err = posix_spawn_file_actions_addclose(&actions, child_out[WRITE]); + err = posix_spawn_file_actions_addclosefrom_np(actions.get(), + STDERR_FILENO + 1); } +#endif if (err == 0) { - err = posix_spawnp(&child, program.c_str(), &actions, &attr, argv.data(), - environ); + err = posix_spawnp(&child, program.c_str(), actions.get(), attr.get(), + argv.data(), environ); } - posix_spawnattr_destroy(&attr); - posix_spawn_file_actions_destroy(&actions); if (err != 0) { - ::close(child_in[READ]); - ::close(child_in[WRITE]); - ::close(child_out[READ]); - ::close(child_out[WRITE]); child = -1; errno = err; throw Error("BlackBoxExec", last_error("starting blackbox process failed")); } - ::close(child_in[READ]); - ::close(child_out[WRITE]); + child_in[READ].reset(); + child_out[WRITE].reset(); - pipe_send = child_in[WRITE]; #ifdef SO_NOSIGPIPE int nosigpipe = 1; - if (setsockopt(pipe_send, SOL_SOCKET, SO_NOSIGPIPE, &nosigpipe, + if (setsockopt(child_in[WRITE].get(), SOL_SOCKET, SO_NOSIGPIPE, &nosigpipe, sizeof(nosigpipe)) != 0) { - ::close(pipe_send); - pipe_send = -1; - ::close(child_out[READ]); + int e = errno; terminate_child(child); child = -1; + errno = e; throw Error("BlackBoxExec", last_error("SO_NOSIGPIPE setup failed")); } #endif - file_receive = fdopen(child_out[READ], "r"); - if (file_receive == NULL) { - ::close(pipe_send); - pipe_send = -1; - ::close(child_out[READ]); + FILE *receive = fdopen(child_out[READ].get(), "r"); + if (receive == NULL) { + int e = errno; terminate_child(child); child = -1; + errno = e; throw Error("BlackBoxExec", last_error("fdopen failed")); } + file_receive = receive; + child_out[READ].release(); + pipe_send = child_in[WRITE].release(); } void diff --git a/gecode/support/config.hpp.in b/gecode/support/config.hpp.in index 1298f113f9..012bffb512 100644 --- a/gecode/support/config.hpp.in +++ b/gecode/support/config.hpp.in @@ -82,6 +82,15 @@ /* Whether we have mtrace for memory leak debugging */ #undef GECODE_HAS_MTRACE +/* Whether posix_spawn_file_actions_addclosefrom_np is available */ +#undef GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP + +/* Whether posix_spawn_file_actions_addinherit_np is available */ +#undef GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP + +/* Whether POSIX_SPAWN_CLOEXEC_DEFAULT is available */ +#undef GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT + /* Whether Qt is available */ #undef GECODE_HAS_QT From 18132519435524db5562b14a9587574486f96061 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Fri, 10 Jul 2026 20:53:02 +0200 Subject: [PATCH 07/14] Finalize native blackbox backends Harden Windows handle and job-object management and adopt the fixed-width, instance-based shared-library ABI. --- CMakeLists.txt | 6 + configure | 10 + configure.ac | 6 + gecode/flatzinc/blackbox.cpp | 769 +++++++++++------- gecode/flatzinc/blackbox.hh | 95 ++- .../blackbox/blackbox_annotations.mzn | 40 +- gecode/flatzinc/registry.cpp | 31 +- gecode/support/config.hpp.in | 3 + 8 files changed, 587 insertions(+), 373 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a9b62b54b3..1c44172a50 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -762,6 +762,12 @@ if(HAVE_POSIX_SPAWN_ADDCLOSEFROM_NP) set(GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP 1) endif() +if((GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT AND + GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP) OR + GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP) + set(GECODE_HAS_POSIX_BLACKBOX_EXEC 1) +endif() + check_c_source_compiles("int main() { return __builtin_ffsll(0); }" HAVE_BUILTIN_FFSLL) if(HAVE_BUILTIN_FFSLL) set(GECODE_HAS_BUILTIN_FFSLL "/**/") diff --git a/configure b/configure index 7b1931d4a4..308eab4c19 100755 --- a/configure +++ b/configure @@ -14554,6 +14554,16 @@ then : printf "%s\n" "#define GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP 1" >>confdefs.h +fi + +if test "x$ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np" = "xyes" || + test "x$ac_cv_gecode_posix_spawn_addclosefrom_np" = "xyes" +then : + + +printf "%s\n" "#define GECODE_HAS_POSIX_BLACKBOX_EXEC 1" >>confdefs.h + + fi GECODE_SAVE_LIBS=${LIBS} diff --git a/configure.ac b/configure.ac index 14f1a145e8..b2c3ef0d58 100644 --- a/configure.ac +++ b/configure.ac @@ -322,6 +322,12 @@ AS_IF([test "x$ac_cv_gecode_posix_spawn_addclosefrom_np" = "xyes"], [ [Whether posix_spawn_file_actions_addclosefrom_np is available]) ]) +AS_IF([test "x$ac_cv_gecode_posix_spawn_cloexec_default_addinherit_np" = "xyes" || + test "x$ac_cv_gecode_posix_spawn_addclosefrom_np" = "xyes"], [ + AC_DEFINE([GECODE_HAS_POSIX_BLACKBOX_EXEC], [1], + [Whether persistent process blackboxes are supported]) +]) + GECODE_SAVE_LIBS=${LIBS} AC_SEARCH_LIBS([dlopen], [dl], [GECODE_DL_LIBS=${ac_cv_search_dlopen}], [GECODE_DL_LIBS=]) diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp index 0653050c2a..b97f5e1dc0 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox.cpp @@ -31,7 +31,16 @@ * */ -#if !defined(_WIN32) && !defined(_GNU_SOURCE) +#if defined(_WIN32) +#if !defined(_WIN32_WINNT) || (_WIN32_WINNT < 0x0600) +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 +#endif +#if !defined(WINVER) || (WINVER < 0x0600) +#undef WINVER +#define WINVER 0x0600 +#endif +#elif !defined(_GNU_SOURCE) #define _GNU_SOURCE 1 #endif @@ -39,10 +48,11 @@ #include #include -#include #include #include #include +#include +#include #include #include #include @@ -61,8 +71,8 @@ #include #else #include +#ifdef GECODE_HAS_POSIX_BLACKBOX_EXEC #include -#include #include #include #include @@ -73,6 +83,7 @@ #include extern char **environ; #endif +#endif #ifdef GECODE_HAS_THREADS #include @@ -107,6 +118,125 @@ windows_error(const std::string &prefix, DWORD err) { return prefix + " (Windows error " + std::to_string(err) + ")"; } +class WindowsHandle { +private: + HANDLE handle; +public: + explicit WindowsHandle(HANDLE handle0=NULL) : handle(handle0) {} + ~WindowsHandle(void) { reset(); } + + WindowsHandle(const WindowsHandle &) = delete; + WindowsHandle &operator=(const WindowsHandle &) = delete; + + HANDLE get(void) const { return handle; } + HANDLE *put(void) { + reset(); + return &handle; + } + HANDLE release(void) { + HANDLE handle0 = handle; + handle = NULL; + return handle0; + } + bool valid(void) const { + return (handle != NULL) && (handle != INVALID_HANDLE_VALUE); + } + void reset(HANDLE handle0=NULL) { + if (valid()) { + CloseHandle(handle); + } + handle = handle0; + } +}; + +class WindowsAttributeList { +private: + std::vector buffer; + LPPROC_THREAD_ATTRIBUTE_LIST list; + bool initialized; +public: + WindowsAttributeList(void) : list(NULL), initialized(false) {} + ~WindowsAttributeList(void) { + if (initialized) { + DeleteProcThreadAttributeList(list); + } + } + + void init(void) { + SIZE_T size = 0; + InitializeProcThreadAttributeList(NULL, 1, 0, &size); + if (size == 0) { + throw Error("BlackBoxExec", + windows_error("ProcThreadAttributeList size query failed", + GetLastError())); + } + buffer.resize(size); + list = reinterpret_cast(buffer.data()); + if (!InitializeProcThreadAttributeList(list, 1, 0, &size)) { + throw Error("BlackBoxExec", + windows_error("InitializeProcThreadAttributeList failed", + GetLastError())); + } + initialized = true; + } + + void set_inherited_handles(HANDLE *handles, DWORD count) { + if (!UpdateProcThreadAttribute(list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + handles, sizeof(HANDLE) * count, NULL, + NULL)) { + throw Error("BlackBoxExec", + windows_error("PROC_THREAD_ATTRIBUTE_HANDLE_LIST failed", + GetLastError())); + } + } + + LPPROC_THREAD_ATTRIBUTE_LIST get(void) const { return list; } +}; + +bool +has_dll_suffix(const std::string &name) { + if (name.size() < 4) { + return false; + } + const char *suffix = ".dll"; + for (size_t i = 0; i < 4; i++) { + if (std::tolower(static_cast(name[name.size() - 4 + i])) != + suffix[i]) { + return false; + } + } + return true; +} + +std::vector +dll_candidates(const std::string &name) { + std::vector candidates; + candidates.push_back(name); + if (!has_dll_suffix(name)) { + candidates.push_back(name + ".dll"); + } + const size_t separator = name.find_last_of("\\/"); + const std::string directory = + (separator == std::string::npos) ? std::string() : + name.substr(0, separator + 1); + const std::string basename = + (separator == std::string::npos) ? name : name.substr(separator + 1); + if (basename.compare(0, 3, "lib") != 0) { + std::string prefixed = directory + "lib" + basename; + if (!has_dll_suffix(prefixed)) { + prefixed += ".dll"; + } + candidates.push_back(prefixed); + } + return candidates; +} + +bool +qualified_path(const std::wstring &program) { + return (program.find_first_of(L"\\/") != std::wstring::npos) || + ((program.size() > 1) && (program[1] == L':')); +} + void close_library(void *library) { if (library != nullptr) { @@ -121,6 +251,7 @@ close_library(void *library) { } } +#ifdef GECODE_HAS_POSIX_BLACKBOX_EXEC int set_cloexec(int fd) { int flags = fcntl(fd, F_GETFD); @@ -298,17 +429,43 @@ send_no_sigpipe(int fd, const char *data, size_t size) { #endif } #endif +#endif -int -checked_int(long long v, const char *source, size_t i) { - if ((v < Int::Limits::min) || (v > Int::Limits::max) || - (v < std::numeric_limits::min()) || - (v > std::numeric_limits::max())) { +template +T +library_symbol(void *library, const char *name, unsigned int stdcall_bytes) { +#ifdef _WIN32 + FARPROC symbol = GetProcAddress(static_cast(library), name); +#if defined(_M_IX86) || defined(__i386__) + if (symbol == nullptr) { + const std::string decorated = + std::string("_") + name + "@" + std::to_string(stdcall_bytes); + symbol = GetProcAddress(static_cast(library), decorated.c_str()); + } + if (symbol == nullptr) { + const std::string decorated = + std::string(name) + "@" + std::to_string(stdcall_bytes); + symbol = GetProcAddress(static_cast(library), decorated.c_str()); + } +#else + (void)stdcall_bytes; +#endif + return reinterpret_cast(symbol); +#else + (void)stdcall_bytes; + T symbol = nullptr; + *(void **)(&symbol) = dlsym(library, name); + return symbol; +#endif +} + +void +check_int(int64_t v, const char *source, size_t i) { + if (!Int::Limits::valid(static_cast(v))) { throw Error("Blackbox", std::string(source) + " integer " + std::to_string(i) + " is outside Gecode's integer range"); } - return static_cast(v); } #ifdef GECODE_HAS_FLOAT_VARS @@ -346,30 +503,41 @@ const size_t max_exec_response_size = 1024 * 1024; } // namespace -BlackBoxLibrary::BlackBoxLibrary(const std::string &name) - : library(nullptr), library_fzn_blackbox(nullptr), - library_fzn_initialize(nullptr) { +#ifdef GECODE_HAS_THREADS +class BlackBoxLibrary::Instance { +public: + std::thread::id owner; + void *value; + Support::Mutex mutex; + + Instance(const std::thread::id &owner0, void *value0) + : owner(owner0), value(value0) {} +}; +#endif + +BlackBoxLibrary::BlackBoxLibrary(const std::string &name, + const std::vector &args) + : library(nullptr), library_fzn_init(nullptr), library_fzn_clone(nullptr), + library_fzn_blackbox(nullptr), library_fzn_free(nullptr), + root_instance(nullptr) +{ std::string loadError; void *loaded = nullptr; #ifdef _WIN32 - DWORD err = 0; - std::wstring wname = utf8_to_wide(name); - loaded = LoadLibraryW(wname.c_str()); - if (!loaded) { - err = GetLastError(); - loadError = std::string("unable to locate library `") + name + "'"; - std::wstring wdll = utf8_to_wide(name + ".dll"); - loaded = LoadLibraryW(wdll.c_str()); - if (!loaded) { - err = GetLastError(); + DWORD err = ERROR_FILE_NOT_FOUND; + std::string failed_candidate = name; + for (const std::string &candidate : dll_candidates(name)) { + loaded = LoadLibraryW(utf8_to_wide(candidate).c_str()); + if (loaded != nullptr) { + break; } + failed_candidate = candidate; + err = GetLastError(); } - if (!loaded) { - std::wstring wlibdll = utf8_to_wide(std::string("lib") + name + ".dll"); - loaded = LoadLibraryW(wlibdll.c_str()); - if (!loaded) { - loadError += " (" + windows_error("LoadLibraryW failed", err) + ")"; - } + if (loaded == nullptr) { + loadError = std::string("unable to locate library `") + name + "' (" + + windows_error("LoadLibraryW failed for `" + failed_candidate + + "'", err) + ")"; } #else loaded = dlopen(name.c_str(), RTLD_LAZY); @@ -393,110 +561,137 @@ BlackBoxLibrary::BlackBoxLibrary(const std::string &name) throw Error("Blackbox", "Unable to open dynamic library: " + loadError); } - // find symbol for blackbox function -#ifdef _WIN32 - library_fzn_blackbox = reinterpret_cast( - GetProcAddress((HMODULE)loaded, "fzn_blackbox")); -#if defined(_M_IX86) || defined(__i386__) - if (!library_fzn_blackbox) { - library_fzn_blackbox = reinterpret_cast( - GetProcAddress((HMODULE)loaded, "_fzn_blackbox@32")); - } - if (!library_fzn_blackbox) { - library_fzn_blackbox = reinterpret_cast( - GetProcAddress((HMODULE)loaded, "fzn_blackbox@32")); - } + bool root_initialized = false; + try { + // fzn_blackbox is the only required entry point. +#ifndef _WIN32 + dlerror(); #endif - std::string symError = "."; + library_fzn_blackbox = + library_symbol(loaded, "fzn_blackbox", + 36); + std::string symError("."); + if (library_fzn_blackbox == nullptr) { +#ifdef _WIN32 + symError += " (" + + windows_error("GetProcAddress failed", GetLastError()) + ")"; #else - *(void **)(&library_fzn_blackbox) = dlsym(loaded, "fzn_blackbox"); - std::string symError(": "); - if (!library_fzn_blackbox) { - symError += std::string(dlerror()); - } + const char *error = dlerror(); + if (error != nullptr) { + symError += std::string(": ") + error; + } #endif - if (!library_fzn_blackbox) { + throw Error("Blackbox", + "Unable to find symbol `fzn_blackbox` in dynamic library" + + symError); + } + + library_fzn_init = + library_symbol(loaded, "fzn_init", 8); + library_fzn_clone = + library_symbol(loaded, "fzn_clone", 4); + library_fzn_free = + library_symbol(loaded, "fzn_free", 4); + if ((library_fzn_init != nullptr) && (library_fzn_clone == nullptr)) { + throw Error("Blackbox", + "Dynamic library exports `fzn_init` but not `fzn_clone`"); + } + if (library_fzn_init != nullptr) { + std::vector argv; + argv.reserve(args.size()); + for (const std::string &arg : args) { + argv.push_back(arg.c_str()); + } + root_instance = library_fzn_init(argv.data(), argv.size()); + root_initialized = true; + } + library = loaded; + } catch (...) { + if (root_initialized && (library_fzn_free != nullptr)) { + try { + library_fzn_free(root_instance); + } catch (...) {} + } close_library(loaded); - throw Error("Blackbox", - "Unable to find symbol `fzn_blackbox` in dynamic library" + - symError); + throw; } +} -#ifdef _WIN32 - char path[MAX_PATH]; - DWORD path_size = GetModuleFileNameA(static_cast(loaded), path, - static_cast(sizeof(path))); - if ((path_size > 0) && (path_size < sizeof(path))) { - library_identity.assign(path, path_size); - } -#else - Dl_info info; - if ((dladdr(reinterpret_cast(library_fzn_blackbox), &info) != 0) && - (info.dli_fname != nullptr)) { - char resolved[PATH_MAX]; - if (realpath(info.dli_fname, resolved) != nullptr) { - library_identity = resolved; - } else { - library_identity = info.dli_fname; +BlackBoxLibrary::~BlackBoxLibrary() { + if ((library_fzn_init != nullptr) && (library_fzn_free != nullptr)) { +#ifdef GECODE_HAS_THREADS + for (Instance *instance : instances) { + try { + library_fzn_free(instance->value); + } catch (...) {} } - } #endif - if (library_identity.empty()) { - library_identity = name; - } - - // Look up the optional initialization function. Calling it is deferred - // until the model-local backend cache has checked its configuration. -#ifdef _WIN32 - library_fzn_initialize = reinterpret_cast( - GetProcAddress((HMODULE)loaded, "fzn_initialize")); -#if defined(_M_IX86) || defined(__i386__) - if (!library_fzn_initialize) { - library_fzn_initialize = reinterpret_cast( - GetProcAddress((HMODULE)loaded, "_fzn_initialize@8")); + try { + library_fzn_free(root_instance); + } catch (...) {} } - if (!library_fzn_initialize) { - library_fzn_initialize = reinterpret_cast( - GetProcAddress((HMODULE)loaded, "fzn_initialize@8")); +#ifdef GECODE_HAS_THREADS + for (Instance *instance : instances) { + delete instance; } #endif -#else - *(void **)(&library_fzn_initialize) = dlsym(loaded, "fzn_initialize"); -#endif - library = loaded; -} - -BlackBoxLibrary::~BlackBoxLibrary() { close_library(library); } -void -BlackBoxLibrary::initialize(const std::vector &args) { - if (library_fzn_initialize != nullptr) { - std::vector argv; - argv.reserve(args.size()); - for (const std::string &a : args) { - argv.push_back(a.c_str()); +#ifdef GECODE_HAS_THREADS +BlackBoxLibrary::Instance * +BlackBoxLibrary::instance(void) { + const std::thread::id owner = std::this_thread::get_id(); + Support::Lock lock(mutex); + for (Instance *instance : instances) { + if (instance->owner == owner) { + return instance; } - library_fzn_initialize(argv.data(), argv.size()); + } + void *value = library_fzn_clone(root_instance); + Instance *clone = nullptr; + try { + clone = new Instance(owner, value); + instances.push_back(clone); + return clone; + } catch (...) { + delete clone; + if (library_fzn_free != nullptr) { + try { + library_fzn_free(value); + } catch (...) {} + } + throw; } } - -const std::string & -BlackBoxLibrary::identity(void) const { - return library_identity; -} +#endif void BlackBoxLibrary::run(const std::vector &int_in, const std::vector &float_in, std::vector &int_out, std::vector &float_out) { - library_fzn_blackbox(int_in.data(), int_in.size(), float_in.data(), - float_in.size(), int_out.data(), int_out.size(), +#ifdef GECODE_HAS_THREADS + if (library_fzn_init != nullptr) { + Instance *selected = this->instance(); + Support::Lock lock(selected->mutex); + library_fzn_blackbox(selected->value, int_in.data(), int_in.size(), + float_in.data(), float_in.size(), int_out.data(), + int_out.size(), float_out.data(), float_out.size()); + } else { + library_fzn_blackbox(nullptr, int_in.data(), int_in.size(), + float_in.data(), float_in.size(), int_out.data(), + int_out.size(), + float_out.data(), float_out.size()); + } +#else + library_fzn_blackbox(root_instance, int_in.data(), int_in.size(), + float_in.data(), float_in.size(), int_out.data(), + int_out.size(), float_out.data(), float_out.size()); +#endif for (size_t i = 0; i < int_out.size(); ++i) { - int_out[i] = checked_int(int_out[i], "library output", i); + check_int(int_out[i], "library output", i); } #ifdef GECODE_HAS_FLOAT_VARS check_floats(float_out, "library output"); @@ -510,7 +705,7 @@ class BlackBoxExec::Session { HANDLE process; HANDLE pipe_send; HANDLE pipe_receive; -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) pid_t child; int pipe_send; FILE *file_receive; @@ -522,8 +717,10 @@ class BlackBoxExec::Session { static std::string last_error(const std::string &prefix) { #ifdef _WIN32 return prefix + " (Windows error " + std::to_string(GetLastError()) + ")"; -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) return prefix + " (errno " + std::to_string(errno) + ")"; +#else + return prefix; #endif } @@ -559,7 +756,7 @@ class BlackBoxExec::Session { void open_windows(const std::string &program, const std::vector &args); void close_windows(void); -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) static void sleep_grace_period(void) { struct timespec remaining = {0, 10000000}; while ((nanosleep(&remaining, &remaining) == -1) && (errno == EINTR)) {} @@ -633,26 +830,25 @@ class BlackBoxExec::Session { public: Session(const std::string &program, const std::vector &args) -#ifdef GECODE_HAS_THREADS -#ifdef _WIN32 - : job(NULL), process(NULL), pipe_send(NULL), pipe_receive(NULL), - owner(std::this_thread::get_id()) -#else - : child(-1), pipe_send(-1), file_receive(NULL), - owner(std::this_thread::get_id()) -#endif -#else #ifdef _WIN32 : job(NULL), process(NULL), pipe_send(NULL), pipe_receive(NULL) -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) : child(-1), pipe_send(-1), file_receive(NULL) -#endif #endif { +#ifdef GECODE_HAS_THREADS + owner = std::this_thread::get_id(); +#endif #ifdef _WIN32 open_windows(program, args); -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) open_posix(program, args); +#else + (void)program; + (void)args; + throw Error("BlackBoxExec", + "Persistent process blackboxes are not supported on this " + "platform"); #endif } @@ -705,7 +901,7 @@ class BlackBoxExec::Session { oss << c[0]; } return oss.str(); -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) const char *p = out_buf.c_str(); size_t remaining = out_buf.size(); while (remaining > 0) { @@ -755,13 +951,18 @@ class BlackBoxExec::Session { } } return in_buffer; +#else + (void)out_buf; + throw Error("BlackBoxExec", + "Persistent process blackboxes are not supported on this " + "platform"); #endif } void close(void) { #ifdef _WIN32 close_windows(); -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) close_posix(); #endif } @@ -782,107 +983,83 @@ BlackBoxExec::Session::open_windows(const std::string &program, std::vector cmdline(prog.begin(), prog.end()); cmdline.push_back(L'\0'); - SIZE_T attr_size = 0; - InitializeProcThreadAttributeList(NULL, 1, 0, &attr_size); - std::vector attr_buf(attr_size); - SECURITY_ATTRIBUTES saAttr; saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); saAttr.bInheritHandle = TRUE; saAttr.lpSecurityDescriptor = NULL; - HANDLE child_stdin_read = NULL; - HANDLE child_stdin_write = NULL; - HANDLE child_stdout_read = NULL; - HANDLE child_stdout_write = NULL; - HANDLE child_stderr_write = NULL; - LPPROC_THREAD_ATTRIBUTE_LIST attr_list = NULL; - - auto close_startup_handles = [&]() { - close_handle(child_stdin_read); - close_handle(child_stdin_write); - close_handle(child_stdout_read); - close_handle(child_stdout_write); - close_handle(child_stderr_write); - }; - auto destroy_attr_list = [&]() { - if (attr_list != NULL) { - DeleteProcThreadAttributeList(attr_list); - attr_list = NULL; - } - }; - - if (!CreatePipe(&child_stdout_read, &child_stdout_write, &saAttr, 0)) { + WindowsHandle child_stdin_read; + WindowsHandle child_stdin_write; + WindowsHandle child_stdout_read; + WindowsHandle child_stdout_write; + WindowsHandle child_stderr_write; + if (!CreatePipe(child_stdout_read.put(), child_stdout_write.put(), &saAttr, + 0)) { throw Error("BlackBoxExec", last_error("Stdout CreatePipe failed")); } - if (!SetHandleInformation(child_stdout_read, HANDLE_FLAG_INHERIT, 0)) { - DWORD err = GetLastError(); - close_startup_handles(); + if (!SetHandleInformation(child_stdout_read.get(), HANDLE_FLAG_INHERIT, 0)) { throw Error("BlackBoxExec", - windows_error("Stdout SetHandleInformation failed", err)); + last_error("Stdout SetHandleInformation failed")); } - if (!CreatePipe(&child_stdin_read, &child_stdin_write, &saAttr, 0)) { - DWORD err = GetLastError(); - close_startup_handles(); - throw Error("BlackBoxExec", windows_error("Stdin CreatePipe failed", err)); + if (!CreatePipe(child_stdin_read.put(), child_stdin_write.put(), &saAttr, + 0)) { + throw Error("BlackBoxExec", last_error("Stdin CreatePipe failed")); } - if (!SetHandleInformation(child_stdin_write, HANDLE_FLAG_INHERIT, 0)) { - DWORD err = GetLastError(); - close_startup_handles(); + if (!SetHandleInformation(child_stdin_write.get(), HANDLE_FLAG_INHERIT, 0)) { throw Error("BlackBoxExec", - windows_error("Stdin SetHandleInformation failed", err)); + last_error("Stdin SetHandleInformation failed")); } HANDLE parent_stderr = GetStdHandle(STD_ERROR_HANDLE); if ((parent_stderr != NULL) && (parent_stderr != INVALID_HANDLE_VALUE)) { if (!DuplicateHandle(GetCurrentProcess(), parent_stderr, - GetCurrentProcess(), &child_stderr_write, 0, TRUE, + GetCurrentProcess(), child_stderr_write.put(), 0, TRUE, DUPLICATE_SAME_ACCESS)) { - DWORD err = GetLastError(); - close_startup_handles(); throw Error("BlackBoxExec", - windows_error("stderr DuplicateHandle failed", err)); + last_error("stderr DuplicateHandle failed")); + } + } else { + HANDLE nul = CreateFileW(L"NUL", GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, &saAttr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (nul == INVALID_HANDLE_VALUE) { + throw Error("BlackBoxExec", last_error("stderr NUL CreateFile failed")); } + child_stderr_write.reset(nul); } + WindowsAttributeList attr_list; + attr_list.init(); PROCESS_INFORMATION piProcInfo; STARTUPINFOEXW siStartInfo; ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); ZeroMemory(&siStartInfo, sizeof(STARTUPINFOEXW)); siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEXW); - siStartInfo.StartupInfo.hStdOutput = child_stdout_write; - siStartInfo.StartupInfo.hStdInput = child_stdin_read; - siStartInfo.StartupInfo.hStdError = child_stderr_write; + siStartInfo.StartupInfo.hStdOutput = child_stdout_write.get(); + siStartInfo.StartupInfo.hStdInput = child_stdin_read.get(); + siStartInfo.StartupInfo.hStdError = child_stderr_write.get(); siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; - HANDLE inherit_handles[3] = {child_stdin_read, child_stdout_write, NULL}; - DWORD inherit_count = 2; - if (child_stderr_write != NULL) { - inherit_handles[inherit_count++] = child_stderr_write; - } + HANDLE inherit_handles[3] = {child_stdin_read.get(), child_stdout_write.get(), + child_stderr_write.get()}; + attr_list.set_inherited_handles(inherit_handles, 3); + siStartInfo.lpAttributeList = attr_list.get(); - attr_list = reinterpret_cast(attr_buf.data()); - siStartInfo.lpAttributeList = attr_list; - if (!InitializeProcThreadAttributeList(attr_list, 1, 0, &attr_size)) { - DWORD err = GetLastError(); - close_startup_handles(); - throw Error("BlackBoxExec", - windows_error("InitializeProcThreadAttributeList failed", err)); + WindowsHandle process_job(CreateJobObjectW(NULL, NULL)); + if (!process_job.valid()) { + throw Error("BlackBoxExec", last_error("CreateJobObject failed")); } - if (!UpdateProcThreadAttribute(attr_list, 0, - PROC_THREAD_ATTRIBUTE_HANDLE_LIST, - inherit_handles, - sizeof(HANDLE) * inherit_count, - NULL, NULL)) { - DWORD err = GetLastError(); - destroy_attr_list(); - close_startup_handles(); - throw Error("BlackBoxExec", - windows_error("PROC_THREAD_ATTRIBUTE_HANDLE_LIST failed", err)); + JOBOBJECT_EXTENDED_LIMIT_INFORMATION job_info; + ZeroMemory(&job_info, sizeof(job_info)); + job_info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (!SetInformationJobObject(process_job.get(), + JobObjectExtendedLimitInformation, &job_info, + sizeof(job_info))) { + throw Error("BlackBoxExec", last_error("SetInformationJobObject failed")); } BOOL processStarted = - CreateProcessW(nullptr, + CreateProcessW(qualified_path(program_w) ? program_w.c_str() : NULL, cmdline.data(), // command line nullptr, // process security attributes nullptr, // primary thread security attributes @@ -892,54 +1069,67 @@ BlackBoxExec::Session::open_windows(const std::string &program, nullptr, // use parent's current directory &siStartInfo.StartupInfo, &piProcInfo); // receives PROCESS_INFORMATION - destroy_attr_list(); if (!processStarted) { - DWORD err = GetLastError(); - close_startup_handles(); throw Error("BlackBoxExec", windows_error("Unable to start program `" + - program + "'", err)); + program + "'", GetLastError())); } - - HANDLE process_job = CreateJobObjectW(NULL, NULL); - if (process_job != NULL) { - JOBOBJECT_EXTENDED_LIMIT_INFORMATION job_info; - ZeroMemory(&job_info, sizeof(job_info)); - job_info.BasicLimitInformation.LimitFlags = - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if (!SetInformationJobObject(process_job, JobObjectExtendedLimitInformation, - &job_info, sizeof(job_info)) || - !AssignProcessToJobObject(process_job, piProcInfo.hProcess)) { - CloseHandle(process_job); - process_job = NULL; + WindowsHandle process_handle(piProcInfo.hProcess); + WindowsHandle thread_handle(piProcInfo.hThread); + if (!AssignProcessToJobObject(process_job.get(), process_handle.get())) { + DWORD err = GetLastError(); + DWORD terminate_err = ERROR_SUCCESS; + if (!TerminateProcess(process_handle.get(), 1)) { + terminate_err = GetLastError(); + } + DWORD wait = WaitForSingleObject(process_handle.get(), 5000); + std::string message = windows_error( + "Unable to assign blackbox process to required job", err); + if (terminate_err != ERROR_SUCCESS) { + message += "; " + windows_error("TerminateProcess cleanup failed", + terminate_err); + } + if (wait == WAIT_FAILED) { + message += "; " + last_error("process cleanup wait failed"); + } else if (wait == WAIT_TIMEOUT) { + message += "; process cleanup timed out"; } + throw Error("BlackBoxExec", message); } - if (ResumeThread(piProcInfo.hThread) == static_cast(-1)) { + if (ResumeThread(thread_handle.get()) == static_cast(-1)) { DWORD err = GetLastError(); - if (process_job != NULL) { - TerminateJobObject(process_job, 1); - } else { - TerminateProcess(piProcInfo.hProcess, 1); - } - WaitForSingleObject(piProcInfo.hProcess, 5000); - CloseHandle(piProcInfo.hThread); - CloseHandle(piProcInfo.hProcess); - close_handle(process_job); - close_startup_handles(); - throw Error("BlackBoxExec", - windows_error("ResumeThread failed for blackbox process", err)); + DWORD terminate_err = ERROR_SUCCESS; + if (!TerminateJobObject(process_job.get(), 1)) { + terminate_err = GetLastError(); + } + HANDLE assigned_job = process_job.release(); + DWORD close_err = ERROR_SUCCESS; + if (!CloseHandle(assigned_job)) { + close_err = GetLastError(); + } + DWORD wait = WaitForSingleObject(process_handle.get(), 5000); + std::string message = windows_error( + "ResumeThread failed for blackbox process", err); + if (terminate_err != ERROR_SUCCESS) { + message += "; " + windows_error("TerminateJobObject cleanup failed", + terminate_err); + } + if (close_err != ERROR_SUCCESS) { + message += "; " + windows_error("job cleanup close failed", close_err); + } + if (wait == WAIT_FAILED) { + message += "; " + last_error("process cleanup wait failed"); + } else if (wait == WAIT_TIMEOUT) { + message += "; process cleanup timed out"; + } + throw Error("BlackBoxExec", message); } - CloseHandle(piProcInfo.hThread); - close_handle(child_stdout_write); - close_handle(child_stdin_read); - close_handle(child_stderr_write); - - pipe_send = child_stdin_write; - pipe_receive = child_stdout_read; - process = piProcInfo.hProcess; - job = process_job; + pipe_send = child_stdin_write.release(); + pipe_receive = child_stdout_read.release(); + process = process_handle.release(); + job = process_job.release(); } void @@ -960,7 +1150,7 @@ BlackBoxExec::Session::close_windows(void) { } close_handle(job); } -#else +#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) void BlackBoxExec::Session::open_posix(const std::string &program, const std::vector &args) { @@ -975,13 +1165,6 @@ BlackBoxExec::Session::open_posix(const std::string &program, } argv.push_back(nullptr); -#if !((defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ - defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP)) || \ - defined(GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP)) - throw Error("BlackBoxExec", - "Persistent process blackboxes require a safe posix_spawn " - "descriptor-inheritance facility on this platform"); -#endif check_sigchld(); FileDescriptor child_in[2]; @@ -1131,21 +1314,8 @@ class BlackBoxState : public SharedHandle::Object { const BlackBoxHandle &handle0) : program(program0), args(args0), handle(handle0) {} }; - class LibraryEntry { - public: - std::string identity; - std::vector args; - std::vector names; - BlackBoxHandle handle; - LibraryEntry(const std::string &identity0, - const std::vector &args0, - const std::string &name, const BlackBoxHandle &handle0) - : identity(identity0), args(args0), names(1, name), handle(handle0) {} - }; - mutable Support::Mutex mutex; std::vector exec; - std::vector library; std::exception_ptr exception; std::atomic error_recorded; @@ -1199,8 +1369,8 @@ BlackBoxHandle BlackBoxState::blackBox(const std::string &mode, const std::string &instantiation, const std::vector &args) { - Support::Lock lock(mutex); if (mode == "exec") { + Support::Lock lock(mutex); for (const ExecEntry &e : exec) { if ((e.program == instantiation) && (e.args == args)) { return e.handle; @@ -1211,34 +1381,7 @@ BlackBoxState::blackBox(const std::string &mode, return handle; } if (mode == "dll") { - for (LibraryEntry &e : library) { - if (std::find(e.names.begin(), e.names.end(), instantiation) != - e.names.end()) { - if (e.args != args) { - throw Error("Blackbox", "Conflicting initialization arguments for " - "dynamic library `" + e.identity + "'"); - } - return e.handle; - } - } - - BlackBoxHandle handle(new BlackBoxLibrary(instantiation)); - BlackBoxLibrary *black_box = - static_cast(handle()); - for (LibraryEntry &e : library) { - if (e.identity == black_box->identity()) { - if (e.args != args) { - throw Error("Blackbox", "Conflicting initialization arguments for " - "dynamic library `" + e.identity + "'"); - } - e.names.push_back(instantiation); - return e.handle; - } - } - black_box->initialize(args); - library.push_back(LibraryEntry(black_box->identity(), args, instantiation, - handle)); - return handle; + return BlackBoxHandle(new BlackBoxLibrary(instantiation, args)); } throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); } @@ -1334,6 +1477,15 @@ void BlackBoxExec::run(const std::vector &int_in, } return q; }; + auto check_integer_tail = [](const char *q, const char *end) { + while (q != end) { + if (*q != ' ' && *q != '\t' && *q != '\r') { + return false; + } + ++q; + } + return true; + }; auto check_number_tail = [](std::istringstream &in) { char c; while (in.get(c)) { @@ -1346,17 +1498,23 @@ void BlackBoxExec::run(const std::vector &int_in, for (size_t i = 0; i < int_out.size(); ++i) { skip_ws(p); const char *end = value_end(p); - std::istringstream in(std::string(p, end)); - in.imbue(std::locale::classic()); - long long v; - if (!(in >> v) || !check_number_tail(in)) { + const char *integer = p; + if (*integer == '+') { + ++integer; + } + int64_t value; + const std::from_chars_result parsed = + std::from_chars(integer, end, value); + if ((parsed.ptr == integer) || (parsed.ec != std::errc()) || + !check_integer_tail(parsed.ptr, end)) { throw Error("BlackBoxExec", "Failed to read output integer " + std::to_string(i) + " from blackbox process output, " + std::to_string(int_out.size()) + " integer values were expected."); } - int_out[i] = checked_int(v, "blackbox process output", i); + check_int(value, "blackbox process output", i); + int_out[i] = value; p = end; skip_ws(p); if (i + 1 < int_out.size()) { @@ -1418,9 +1576,8 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { ) { std::vector int_in(int_input.size()); std::vector int_out(int_output.size()); - // std::cerr << "Black Box Fn input: "; for (int i = 0; i < int_in.size(); i++) { - int_in[i] = int_input[i].val(); + int_in[i] = static_cast(int_input[i].val()); } std::vector float_in; std::vector float_out; @@ -1440,7 +1597,6 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { } for (int i = 0; i < int_out.size(); i++) { - // std::cerr << int_out[i] << " "; GECODE_ME_CHECK(int_output[i].eq(home, static_cast(int_out[i]))); } #ifdef GECODE_HAS_FLOAT_VARS @@ -1454,13 +1610,18 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { return ES_FIX; } -ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { +ExecStatus +BlackBoxBounds::evaluate(Home home, ViewArray &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray &fvar, +#endif + BlackBoxHandle &black_box, + const BlackBoxStateHandle &black_box_state) { std::vector int_in(ivar.size() * 2); std::vector int_out(ivar.size() * 2); - // std::cerr << "Black Box Bounds Fn input: "; for (int i = 0; i < ivar.size(); i++) { - int_in[i*2] = ivar[i].min(); - int_in[i*2+1] = ivar[i].max(); + int_in[i*2] = static_cast(ivar[i].min()); + int_in[i*2+1] = static_cast(ivar[i].max()); } std::vector float_in; std::vector float_out; @@ -1481,18 +1642,30 @@ ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { } for (int i = 0; i < ivar.size(); i++) { - // std::cerr << int_out[i*2] << ".." << int_out[i*2+1] << " "; - GECODE_ME_CHECK(ivar[i].gq(home, static_cast(int_out[i*2]))); - GECODE_ME_CHECK(ivar[i].lq(home, static_cast(int_out[i*2+1]))); + if (me_failed(ivar[i].gq(home, static_cast(int_out[i*2]))) || + me_failed(ivar[i].lq(home, static_cast(int_out[i*2+1])))) { + return ES_FAILED; + } } #ifdef GECODE_HAS_FLOAT_VARS for (int i = 0; i < fvar.size(); i++) { - GECODE_ME_CHECK(fvar[i].gq(home, float_out[i*2])); - GECODE_ME_CHECK(fvar[i].lq(home, float_out[i*2+1])); + if (me_failed(fvar[i].gq(home, float_out[i*2])) || + me_failed(fvar[i].lq(home, float_out[i*2+1]))) { + return ES_FAILED; + } } #endif - return ES_NOFIX; + return ES_OK; +} + +ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { + ExecStatus es = evaluate(home, ivar, +#ifdef GECODE_HAS_FLOAT_VARS + fvar, +#endif + black_box, black_box_state); + return (es == ES_OK) ? ES_NOFIX : es; } void blackbox(Home home, SharedHandle &black_box_state, diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index b85449eada..d4351dd1a9 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -45,6 +45,9 @@ #ifdef GECODE_HAS_FLOAT_VARS #include #endif +#ifdef GECODE_HAS_THREADS +#include +#endif #ifdef _WIN32 #define GECODE_BLACKBOX_CALL __stdcall @@ -74,16 +77,18 @@ public: /// Implementation of a black box function that dynamically loads a library and /// run a contained function. /// -/// The native library entry points can be called concurrently by parallel -/// search workers and must therefore be thread-safe. -class BlackBoxLibrary : public BlackBoxFn { +/// A library backend belongs to one blackbox constraint. If the library exports +/// fzn_init, it creates a root instance for that constraint. With threads, the +/// root is a prototype: each calling thread receives and reuses its own clone, +/// and the same clone is never used by concurrent calls. Without threads, +/// fzn_blackbox receives the root instance. A library without fzn_init is +/// stateless: fzn_blackbox receives a null instance and can be called +/// concurrently. +class GECODE_FLATZINC_EXPORT BlackBoxLibrary : public BlackBoxFn { public: - BlackBoxLibrary(const std::string &name); + BlackBoxLibrary(const std::string &name, + const std::vector &args); ~BlackBoxLibrary(); - /// Initialize the library with the model-specific configuration - void initialize(const std::vector &args); - /// Return the loaded library identity - const std::string &identity(void) const; void run(const std::vector &int_in, const std::vector &float_in, std::vector &int_out, @@ -91,11 +96,23 @@ public: protected: void *library; - std::string library_identity; - void(GECODE_BLACKBOX_CALL *library_fzn_blackbox)( - const int64_t *, size_t, const double *, size_t, int64_t *, size_t, - double *, size_t); - void(GECODE_BLACKBOX_CALL *library_fzn_initialize)(const char **, size_t); + void *(GECODE_BLACKBOX_CALL *library_fzn_init)(const char **, size_t); + void *(GECODE_BLACKBOX_CALL *library_fzn_clone)(void *); + void (GECODE_BLACKBOX_CALL *library_fzn_blackbox)( + void *, const int64_t *, size_t, const double *, size_t, int64_t *, + size_t, double *, size_t); + void (GECODE_BLACKBOX_CALL *library_fzn_free)(void *); + void *root_instance; + +#ifdef GECODE_HAS_THREADS + class Instance; + /// Mutex protecting the worker-to-instance table. + Support::Mutex mutex; + /// Cloned instances, one for each calling worker. + std::vector instances; + + Instance *instance(void); +#endif }; /// Implementation of a blackbox function that starts a separate process to @@ -104,12 +121,13 @@ protected: /// Parallel search workers do not share a process stream: each calling thread /// gets its own persistent process session, created lazily and reused by that /// thread until the shared blackbox object is destroyed. -class BlackBoxExec : public BlackBoxFn { +class GECODE_FLATZINC_EXPORT BlackBoxExec : public BlackBoxFn { public: BlackBoxExec(const std::string &program, const std::vector &args); ~BlackBoxExec(); void run(const std::vector &int_in, - const std::vector &float_in, std::vector &int_out, + const std::vector &float_in, + std::vector &int_out, std::vector &float_out) override; protected: @@ -270,8 +288,8 @@ public: && (float_input.size() == 0) #endif ) { - std::vector int_in; - std::vector int_out(int_output.size()); + std::vector int_in; + std::vector int_out(int_output.size()); std::vector float_in; std::vector float_out; #ifdef GECODE_HAS_FLOAT_VARS @@ -279,7 +297,8 @@ public: #endif black_box_handle()->run(int_in, float_in, int_out, float_out); for (int i = 0; i < int_output.size(); i++) { - if (me_failed(int_output[i].eq(home, int_out[i]))) { + if (me_failed(int_output[i].eq( + home, static_cast(int_out[i])))) { return ES_FAILED; } } @@ -434,6 +453,13 @@ public: ExecStatus propagate(Space &home, const ModEventDelta &) override; + static ExecStatus evaluate(Home home, ViewArray &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray &fvar, +#endif + BlackBoxHandle &black_box, + const BlackBoxStateHandle &black_box_state); + Propagator *copy(Space &home) override { return new (home) BlackBoxBounds(home, *this); } @@ -462,38 +488,11 @@ public: } #endif if (!has_subscription) { - std::vector int_in(ivar.size() * 2); - std::vector int_out(ivar.size() * 2); - for (int i = 0; i < ivar.size(); i++) { - int_in[i*2] = ivar[i].min(); - int_in[i*2+1] = ivar[i].max(); - } - std::vector float_in; - std::vector float_out; + return evaluate(home, ivar, #ifdef GECODE_HAS_FLOAT_VARS - float_in.resize(fvar.size() * 2); - float_out.resize(fvar.size() * 2); - for (int i = 0; i < fvar.size(); i++) { - float_in[i*2] = fvar[i].min(); - float_in[i*2+1] = fvar[i].max(); - } + fvar, #endif - black_box_handle()->run(int_in, float_in, int_out, float_out); - for (int i = 0; i < ivar.size(); i++) { - if (me_failed(ivar[i].gq(home, int_out[i*2])) || - me_failed(ivar[i].lq(home, int_out[i*2+1]))) { - return ES_FAILED; - } - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < fvar.size(); i++) { - if (me_failed(fvar[i].gq(home, float_out[i*2])) || - me_failed(fvar[i].lq(home, float_out[i*2+1]))) { - return ES_FAILED; - } - } -#endif - return ES_OK; + black_box_handle, black_box_state); } new (home) BlackBoxBounds(home, ivar, diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn index 9b5f8df903..3f95dbebde 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn @@ -1,21 +1,35 @@ % Blackbox annotations execute user-provided code. Use them only for trusted % models and trusted executable/library paths. % -% blackbox_dll loads a native library and calls: +% blackbox_dll loads a native library with the following C ABI: % +% extern "C" { +% void* fzn_init(const char** args, size_t n_args); +% void* fzn_clone(void* instance); % void fzn_blackbox( +% void* instance, % const int64_t* int_in, size_t n_int_in, % const double* float_in, size_t n_float_in, % int64_t* int_out, size_t n_int_out, -% double* float_out, size_t n_float_out) -% -% The library may also export: -% -% void fzn_initialize(const char** args, size_t n_args) -% -% On Windows these functions use the Gecode blackbox calling convention. Native -% library blackboxes can be called concurrently by parallel search workers and -% must be thread-safe. +% double* float_out, size_t n_float_out); +% void fzn_free(void* instance); +% } +% +% fzn_blackbox is required. fzn_init, fzn_clone, and fzn_free are optional; +% exporting fzn_init requires exporting fzn_clone. If fzn_init is absent, the +% library is stateless: fzn_blackbox receives NULL and can be called +% concurrently. If fzn_free is absent, Gecode does not release library state. +% +% Gecode calls fzn_init once for each blackbox_dll constraint, with the +% annotation arguments. In threaded builds its result is a prototype: each +% calling thread receives and reuses one fzn_clone result, and calls using the +% same clone are serialized. Different clones can be called concurrently. In +% builds without threads, fzn_blackbox receives the fzn_init result directly. +% When fzn_free is exported, Gecode calls it once for every fzn_init and +% fzn_clone result: once for the root and once for each clone. +% +% On Windows, export all functions with __declspec(dllexport) and use +% __stdcall. On other systems, use the ordinary C calling convention. % % blackbox_exec starts a persistent subprocess. For each call, Gecode writes one % line to stdin: @@ -29,9 +43,9 @@ % otherwise it can block the solver. During parallel search each worker thread % gets its own process. % -% All blackboxes must be deterministic from FlatZinc's point of view: the same -% input arrays must produce the same output arrays. Implementations may cache -% internally as long as the observable result does not depend on call order. +% All blackboxes must be deterministic from FlatZinc's point of view: equal +% input arrays must produce equal output arrays, regardless of call order or +% internal instance-state evolution. annotation blackbox_dll(string: library); annotation blackbox_dll(string: library, array[int] of string: args); diff --git a/gecode/flatzinc/registry.cpp b/gecode/flatzinc/registry.cpp index cc83ec1e74..29162dbab8 100755 --- a/gecode/flatzinc/registry.cpp +++ b/gecode/flatzinc/registry.cpp @@ -1668,13 +1668,11 @@ namespace Gecode { namespace FlatZinc { void blackbox_source(AST::Node* ann, std::string& mode, std::string& instantiation, std::vector& args) { - auto error = [](const std::string& message) { - throw FlatZinc::Error("Registry", message); - }; - auto string_arg = [&](AST::Node* n, const char* what) { + auto string_arg = [](AST::Node* n, const char* what) { if ((n == nullptr) || !n->isString()) { - error(std::string("Malformed blackbox annotation: ") + what + - " must be a string."); + throw FlatZinc::Error("Registry", + std::string("Malformed blackbox annotation: ") + + what + " must be a string."); } return n->getString(); }; @@ -1682,7 +1680,8 @@ namespace Gecode { namespace FlatZinc { bool has_dll = (ann != nullptr) && ann->hasCall("blackbox_dll"); bool has_exec = (ann != nullptr) && ann->hasCall("blackbox_exec"); if (has_dll && has_exec) { - error("Blackbox constraint has multiple execution method annotations."); + throw FlatZinc::Error( + "Registry", "Blackbox constraint has multiple execution method annotations."); } else if (has_dll) { c = ann->getCall("blackbox_dll"); mode = "dll"; @@ -1690,23 +1689,27 @@ namespace Gecode { namespace FlatZinc { c = ann->getCall("blackbox_exec"); mode = "exec"; } else { - error("Blackbox constraint is missing a valid annotation specifying " - "execution method."); + throw FlatZinc::Error( + "Registry", "Blackbox constraint is missing a valid annotation specifying " + "execution method."); } if ((c == nullptr) || (c->args == nullptr)) { - error("Malformed blackbox annotation: missing target."); + throw FlatZinc::Error("Registry", + "Malformed blackbox annotation: missing target."); } // For a single-argument call `args` is the bare argument node; for the // `(target, args)` form it is an array of the two arguments. if (AST::Array* arr = dynamic_cast(c->args)) { if (arr->a.size() != 2) { - error("Malformed blackbox annotation: expected a target string and " - "an argument array."); + throw FlatZinc::Error( + "Registry", "Malformed blackbox annotation: expected a target string and " + "an argument array."); } instantiation = string_arg(arr->a[0], "target"); if (!arr->a[1]->isArray()) { - error("Malformed blackbox annotation: argument list must be an array " - "of strings."); + throw FlatZinc::Error( + "Registry", "Malformed blackbox annotation: argument list must be an array " + "of strings."); } AST::Array* al = arr->a[1]->getArray(); for (unsigned int i = 0; i < al->a.size(); i++) { diff --git a/gecode/support/config.hpp.in b/gecode/support/config.hpp.in index 012bffb512..9c9588878f 100644 --- a/gecode/support/config.hpp.in +++ b/gecode/support/config.hpp.in @@ -82,6 +82,9 @@ /* Whether we have mtrace for memory leak debugging */ #undef GECODE_HAS_MTRACE +/* Whether persistent process blackboxes are supported */ +#undef GECODE_HAS_POSIX_BLACKBOX_EXEC + /* Whether posix_spawn_file_actions_addclosefrom_np is available */ #undef GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP From 5f6610a28b7f4eacadd53cdc947d8b53ecb813f1 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Fri, 10 Jul 2026 20:53:02 +0200 Subject: [PATCH 08/14] Add cross-platform blackbox integration coverage Exercise native process and library backends across CMake and Autoconf, including concurrency, malformed responses, portability, and diagnostics. --- CMakeLists.txt | 57 +- Makefile.in | 69 +- cmake/GecodeSources.cmake | 3 + configure | 1 + gecode/flatzinc.hh | 5 +- gecode/flatzinc/blackbox.cpp | 13 +- gecode/flatzinc/blackbox.hh | 16 +- gecode/flatzinc/flatzinc.cpp | 12 +- .../blackbox/blackbox_annotations.mzn | 3 + gecode/flatzinc/registry.cpp | 4 +- test/flatzinc.cpp | 57 +- test/flatzinc.hh | 14 +- test/flatzinc/blackbox-dll.cpp | 194 ++++++ test/flatzinc/blackbox-exec.cpp | 268 ++++++++ test/flatzinc/blackbox.cpp | 636 ++++++++++++++---- 15 files changed, 1156 insertions(+), 196 deletions(-) create mode 100644 test/flatzinc/blackbox-dll.cpp create mode 100644 test/flatzinc/blackbox-exec.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c44172a50..66e5236f47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1371,6 +1371,16 @@ if(BUILD_TESTING) set(GECODE_CAN_BUILD_TESTS FALSE) endif() + if(GECODE_ENABLE_FLATZINC) + add_executable(gecode-test-blackbox-exec + ${GECODE_TEST_BLACKBOX_EXEC_SOURCE}) + target_compile_features(gecode-test-blackbox-exec PRIVATE cxx_std_17) + + add_library(gecode-test-blackbox-dll SHARED + ${GECODE_TEST_BLACKBOX_DLL_SOURCE}) + target_compile_features(gecode-test-blackbox-dll PRIVATE cxx_std_17) + endif() + set(GECODE_TEST_SOURCES_SELECTED ${GECODE_TEST_SOURCES}) if(NOT GECODE_ENABLE_SET_VARS) list(FILTER GECODE_TEST_SOURCES_SELECTED EXCLUDE REGEX "^test/set(/|\\.cpp)") @@ -1394,6 +1404,18 @@ if(BUILD_TESTING) ${GECODE_FAULT_TEST_SOURCES}) target_link_libraries(gecode-fault-test PRIVATE ${GECODE_TEST_LINK_LIBS}) endif() + if(GECODE_ENABLE_FLATZINC) + add_dependencies(gecode-test + gecode-test-blackbox-exec + gecode-test-blackbox-dll) + set(GECODE_TEST_BLACKBOX_LOG + "${CMAKE_CURRENT_BINARY_DIR}/gecode-test-blackbox") + set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_CLEAN_FILES + "${GECODE_TEST_BLACKBOX_LOG}.exec_parallel" + "${GECODE_TEST_BLACKBOX_LOG}.exec_descendant" + "${GECODE_TEST_BLACKBOX_LOG}.dll_model" + "${GECODE_TEST_BLACKBOX_LOG}.dll_parallel") + endif() set(GECODE_CHECK_TESTS Branch::Int::Dense::3 @@ -1409,7 +1431,9 @@ if(BUILD_TESTING) NoGoods::Queens Search::DFS::Sol::Binary::Nary::Binary::1::1::1) if(GECODE_ENABLE_FLATZINC) - list(INSERT GECODE_CHECK_TESTS 1 FlatZinc::magic_square) + list(INSERT GECODE_CHECK_TESTS 1 + FlatZinc::magic_square + FlatZinc::blackbox) endif() if(GECODE_ENABLE_SET_VARS) list(APPEND GECODE_CHECK_TESTS @@ -1466,6 +1490,14 @@ if(BUILD_TESTING) add_test(NAME test COMMAND gecode-test ${GECODE_CHECK_ARGS}) set_tests_properties(test PROPERTIES FIXTURES_REQUIRED gecode-test-built) + if(GECODE_ENABLE_FLATZINC) + set(GECODE_TEST_BLACKBOX_ENV + "GECODE_TEST_BLACKBOX_EXEC=$" + "GECODE_TEST_BLACKBOX_DLL=$" + "GECODE_TEST_BLACKBOX_LOG=${GECODE_TEST_BLACKBOX_LOG}") + set_tests_properties(test PROPERTIES + ENVIRONMENT "${GECODE_TEST_BLACKBOX_ENV}") + endif() set(GECODE_CHECK_DEPENDS gecode-test) if(GECODE_ENABLE_FLOAT_VARS) add_custom_target(verify-gecode-check-tests @@ -1477,10 +1509,24 @@ if(BUILD_TESTING) VERBATIM) list(APPEND GECODE_CHECK_DEPENDS verify-gecode-check-tests) endif() - add_custom_target(check - COMMAND $ ${GECODE_CHECK_ARGS} - DEPENDS ${GECODE_CHECK_DEPENDS} - USES_TERMINAL) + if(GECODE_ENABLE_FLATZINC) + list(APPEND GECODE_CHECK_DEPENDS + gecode-test-blackbox-exec + gecode-test-blackbox-dll) + add_custom_target(check + COMMAND ${CMAKE_COMMAND} -E env + GECODE_TEST_BLACKBOX_EXEC=$ + GECODE_TEST_BLACKBOX_DLL=$ + GECODE_TEST_BLACKBOX_LOG=${GECODE_TEST_BLACKBOX_LOG} + $ ${GECODE_CHECK_ARGS} + DEPENDS ${GECODE_CHECK_DEPENDS} + USES_TERMINAL) + else() + add_custom_target(check + COMMAND $ ${GECODE_CHECK_ARGS} + DEPENDS ${GECODE_CHECK_DEPENDS} + USES_TERMINAL) + endif() if(GECODE_ENABLE_FAULT_INJECTION) set(GECODE_FAULT_CHECK_ARGS -iter 1 -threads 1 -test "^Fault::") add_test(NAME fault COMMAND gecode-fault-test ${GECODE_FAULT_CHECK_ARGS}) @@ -1583,6 +1629,7 @@ if(GECODE_INSTALL) PATTERN "**.hpp" PATTERN "LICENSE_1_0.txt" PATTERN "mznlib" EXCLUDE + PATTERN "flatzinc/blackbox.hh" EXCLUDE PATTERN "exampleplugin" EXCLUDE PATTERN "standalone-example" EXCLUDE PATTERN "abi*" EXCLUDE) diff --git a/Makefile.in b/Makefile.in index 4e9654cc38..e0fa9a7dde 100755 --- a/Makefile.in +++ b/Makefile.in @@ -825,11 +825,14 @@ FLATZINCSRC0 = flatzinc.cpp registry.cpp branch.cpp blackbox.cpp FLATZINC_GENSRC0 = parser.tab.cpp lexer.yy.cpp FLATZINCHDR0 = ast.hh conexpr.hh option.hh parser.hh \ plugin.hh registry.hh symboltable.hh varspec.hh \ - branch.hh branch.hpp lastval.hh complete.hh blackbox.hh + branch.hh branch.hpp lastval.hh complete.hh +FLATZINCPRIVATEHDR0 = blackbox.hh FLATZINCSRC = $(FLATZINCSRC0:%=gecode/flatzinc/%) FLATZINC_GENSRC = $(FLATZINC_GENSRC0:%=gecode/flatzinc/%) FLATZINCHDR = $(FLATZINCHDR0:%=gecode/flatzinc/%) gecode/flatzinc.hh +FLATZINCPRIVATEHDR = $(FLATZINCPRIVATEHDR0:%=gecode/flatzinc/%) +FLATZINCALLHDR = $(FLATZINCHDR) $(FLATZINCPRIVATEHDR) FLATZINCOBJ = \ $(FLATZINCSRC:%.cpp=%$(OBJSUFFIX)) \ @@ -1251,6 +1254,10 @@ SEARCHTESTSRC0 = \ ARRAYTESTSRC0 = \ test/array.cpp +BLACKBOXEXECSRC = test/flatzinc/blackbox-exec.cpp +BLACKBOXDLLSRC = test/flatzinc/blackbox-dll.cpp +BLACKBOXSRC = $(BLACKBOXEXECSRC) $(BLACKBOXDLLSRC) + TESTSRC0 = test/test.cpp test/afc.cpp test/ldsb.cpp test/region.cpp \ test/groups.cpp # FailPoint is CMake-only; keep the Autoconf test executable fault-free. @@ -1258,7 +1265,7 @@ TESTSRC0 = test/test.cpp test/afc.cpp test/ldsb.cpp test/region.cpp \ TESTSRC = \ $(TESTSRC0) $(INTTESTSRC0) $(SETTESTSRC0) $(FLOATTESTSRC0) \ $(BRANCHTESTSRC0) $(SEARCHTESTSRC0) \ - $(ARRAYTESTSRC0) $(FLATZINCTESTSRC0) + $(ARRAYTESTSRC0) $(FLATZINCTESTSRC0) $(BLACKBOXSRC) TESTHDR0 = \ test.hh test.hpp int.hh int.hpp set.hh set.hpp float.hh float.hpp \ @@ -1272,6 +1279,18 @@ TESTOBJ = $(INTTESTOBJ) $(SETTESTOBJ) $(FLOATTESTOBJ) \ $(TESTSRC0:%.cpp=%$(OBJSUFFIX)) TESTSBJ = $(TESTOBJ:%$(OBJSUFFIX)=%$(SBJSUFFIX)) TESTEXE = test/test$(EXESUFFIX) +BLACKBOXEXECOBJ = $(BLACKBOXEXECSRC:%.cpp=%$(OBJSUFFIX)) +BLACKBOXDLLOBJ = $(BLACKBOXDLLSRC:%.cpp=%$(OBJSUFFIX)) +BLACKBOXEXEC = test/flatzinc/blackbox-exec$(EXESUFFIX) +BLACKBOXDLL = test/flatzinc/blackbox-dll$(DLLSUFFIX) +BLACKBOXDLLLIB = $(BLACKBOXDLL:%$(DLLSUFFIX)=%$(LIBSUFFIX)) +BLACKBOXLOG = $(abspath test/flatzinc/blackbox) +ifeq "@enable_flatzinc@" "yes" +BLACKBOXFIXTURES = $(BLACKBOXEXEC) $(BLACKBOXDLL) +else +BLACKBOXFIXTURES = +endif + TESTBUILDDIRS = \ test test/int test/set test/float \ test/branch test/assign \ @@ -1304,7 +1323,7 @@ compilelib: mkcompiledirs compileexamples: $(EXAMPLEEXE) -test: mkcompiledirs +test: mkcompiledirs $(BLACKBOXFIXTURES) @$(MAKE) $(VARIMP) $(TESTEXE) CHECKTESTS = Branch::Int::Dense::3 \ @@ -1325,6 +1344,14 @@ CHECKTESTS = Branch::Int::Dense::3 \ Set::Sequence::SeqU1 \ Set::Wait +ifeq "@enable_flatzinc@" "yes" +CHECKTESTS += FlatZinc::blackbox +BLACKBOXCHECKENV = \ + GECODE_TEST_BLACKBOX_EXEC=$(abspath $(BLACKBOXEXEC)) \ + GECODE_TEST_BLACKBOX_DLL=$(abspath $(BLACKBOXDLL)) \ + GECODE_TEST_BLACKBOX_LOG=$(BLACKBOXLOG) +endif + ifeq "@enable_float_vars@" "yes" FLOATCHECKTESTS = Float::Arithmetic::PositiveNRootBounds \ Float::Arithmetic::PowConsistency \ @@ -1352,7 +1379,8 @@ check: test exit 1; \ fi; \ done - $(RUNENVIRONMENT) $(TESTEXE) -iter 2 -threads 0 -fixprob 1 \ + $(BLACKBOXCHECKENV) $(RUNENVIRONMENT) \ + $(TESTEXE) -iter 2 -threads 0 -fixprob 1 \ $(CHECKTESTS:%=-test %) .PHONY: regenerate @@ -1733,7 +1761,7 @@ $(GISTDLL) $(GISTLIB): $(GISTOBJ) $(GISTRES) \ -outputresource:$(GISTDLL)\;2 ifeq "@enable_resource@" "yes" $(FLATZINCRC): - $(RCGEN) $(FLATZINCDLL) $(FLATZINCHDR) $(FLATZINCSRC) > $@ + $(RCGEN) $(FLATZINCDLL) $(FLATZINCALLHDR) $(FLATZINCSRC) > $@ endif $(FLATZINCDLL) $(FLATZINCLIB): $(FLATZINCOBJ) $(FLATZINCRES) \ $(SUPPORTDLL) $(KERNELDLL) $(SEARCHDLL) $(INTDLL) \ @@ -1846,6 +1874,22 @@ $(TESTEXE): $(TESTOBJ) $(TESTRES) $(ALLLIB) $(FIXMANIFEST) $@.manifest $(DLLSUFFIX) $(MANIFEST) -manifest $@.manifest -outputresource:$@\;1 +$(BLACKBOXEXECOBJ) $(BLACKBOXDLLOBJ): | mkcompiledirs + +$(BLACKBOXEXEC): $(BLACKBOXEXECOBJ) + $(CXX) @EXEOUTPUT@$@ $(BLACKBOXEXECOBJ) $(CXXFLAGS) + +ifeq "$(DLLSUFFIX)" "$(LIBSUFFIX)" +$(BLACKBOXDLL): $(BLACKBOXDLLOBJ) + $(CXX) $(DLLFLAGS) $(BLACKBOXDLLOBJ) @LINKOUTPUT@$@ +else +$(BLACKBOXDLL) $(BLACKBOXDLLLIB): $(BLACKBOXDLLOBJ) + $(CXX) $(DLLFLAGS) $(BLACKBOXDLLOBJ) @LINKOUTPUT@$(BLACKBOXDLL) $(GLDFLAGS) + $(FIXMANIFEST) $(BLACKBOXDLL).manifest + $(MANIFEST) -manifest $(BLACKBOXDLL).manifest \ + -outputresource:$(BLACKBOXDLL)\;2 +endif + .PHONY: flatzinc ifeq "@enable_flatzinc@" "yes" flatzinc: $(FLATZINCEXE) @@ -2095,6 +2139,13 @@ clean: changelog.hh doxygen.hh license.hh header.html $(RMF) $(ALLOBJ) $(ALLSBJ) $(ALLOBJ:%$(OBJSUFFIX)=%.pdb) $(RMF) $(TESTOBJ) $(TESTSBJ) $(TESTOBJ:%$(OBJSUFFIX)=%.pdb) + $(RMF) $(BLACKBOXEXECOBJ) $(BLACKBOXDLLOBJ) \ + $(BLACKBOXEXECOBJ:%$(OBJSUFFIX)=%.pdb) \ + $(BLACKBOXDLLOBJ:%$(OBJSUFFIX)=%.pdb) + $(RMF) $(BLACKBOXEXECOBJ:%$(OBJSUFFIX)=%.gcno) \ + $(BLACKBOXDLLOBJ:%$(OBJSUFFIX)=%.gcno) + $(RMF) $(BLACKBOXEXECOBJ:%$(OBJSUFFIX)=%.gcda) \ + $(BLACKBOXDLLOBJ:%$(OBJSUFFIX)=%.gcda) $(RMF) $(GISTMOCSRC) $(RMF) $(LIBTARGETS:%$(DLLSUFFIX)=%$(MANIFESTSUFFIX)) \ $(LIBTARGETS:%$(DLLSUFFIX)=%$(RCSUFFIX)) \ @@ -2104,6 +2155,13 @@ clean: $(EXAMPLEEXE:%=%.rc) $(EXAMPLEEXE:%=%.res) $(RMF) $(TESTEXE:%.exe=%.pdb) $(TESTEXE:%=%.manifest) \ $(TESTEXE:%=%.rc) $(TESTEXE:%=%.res) + $(RMF) $(BLACKBOXEXEC:%.exe=%.pdb) $(BLACKBOXEXEC:%=%.manifest) \ + $(BLACKBOXDLL:%$(DLLSUFFIX)=%$(LIBSUFFIX)) \ + $(BLACKBOXDLL:%$(DLLSUFFIX)=%$(PDBSUFFIX)) \ + $(BLACKBOXDLL:%$(DLLSUFFIX)=%$(EXPSUFFIX)) \ + $(BLACKBOXDLL:%$(DLLSUFFIX)=%$(MANIFESTSUFFIX)) + $(RMF) $(BLACKBOXLOG).exec_parallel $(BLACKBOXLOG).exec_descendant \ + $(BLACKBOXLOG).dll_model $(BLACKBOXLOG).dll_parallel $(RMF) $(FLATZINCEXE:%.exe=%.pdb) $(FLATZINCEXE:%=%.manifest) \ $(FLATZINCEXE:%=%.rc) $(FLATZINCEXE:%=%.res) @@ -2115,6 +2173,7 @@ veryclean: clean $(LIBTARGETS:%$(DLLSUFFIX)=%$(SOSUFFIX)) $(RMF) $(EXAMPLEEXE) $(RMF) $(TESTEXE) + $(RMF) $(BLACKBOXFIXTURES) $(RMF) $(FLATZINCEXE) $(RMF) doc ChangeLog $(RMF) $(ALLOBJ:%$(OBJSUFFIX)=%.gcno) $(TESTOBJ:%$(OBJSUFFIX)=%.gcno) diff --git a/cmake/GecodeSources.cmake b/cmake/GecodeSources.cmake index 4251a0b84e..6a5e7cb041 100644 --- a/cmake/GecodeSources.cmake +++ b/cmake/GecodeSources.cmake @@ -424,4 +424,7 @@ set(GECODE_TEST_SOURCES test/test.cpp ) +set(GECODE_TEST_BLACKBOX_EXEC_SOURCE test/flatzinc/blackbox-exec.cpp) +set(GECODE_TEST_BLACKBOX_DLL_SOURCE test/flatzinc/blackbox-dll.cpp) + set(GECODE_FLATZINC_EXE_SOURCE tools/flatzinc/fzn-gecode.cpp) diff --git a/configure b/configure index 308eab4c19..0a59cb0548 100755 --- a/configure +++ b/configure @@ -37,6 +37,7 @@ esac fi + # Reset variables that may have inherited troublesome values from # the environment. diff --git a/gecode/flatzinc.hh b/gecode/flatzinc.hh index 4f38d5dd85..12375e18b2 100755 --- a/gecode/flatzinc.hh +++ b/gecode/flatzinc.hh @@ -415,6 +415,7 @@ namespace Gecode { namespace FlatZinc { extern Rnd defrnd; class FlatZincSpaceInitData; + class BlackBoxAccess; /** * \brief A space that can be initialized with a %FlatZinc model @@ -462,6 +463,7 @@ namespace Gecode { namespace FlatZinc { /// Copy constructor FlatZincSpace(FlatZincSpace&); private: + friend class BlackBoxAccess; /// Run the search engine template class Engine> void @@ -604,9 +606,6 @@ namespace Gecode { namespace FlatZinc { /// Post a constraint specified by \a ce void postConstraints(std::vector& ces); - /// Return opaque state used while posting blackbox constraints - SharedHandle& blackBoxState(void); - /// Post the solve item void solve(AST::Array* annotation); /// Post that integer variable \a var should be minimized diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox.cpp index b97f5e1dc0..589a8bed17 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox.cpp @@ -886,6 +886,10 @@ class BlackBoxExec::Session { DWORD count = 0; BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); if (!success) { + if (GetLastError() == ERROR_BROKEN_PIPE) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } throw Error( "BlackBoxExec", "Failed to read blackbox process output from pipe"); @@ -1071,8 +1075,9 @@ BlackBoxExec::Session::open_windows(const std::string &program, &piProcInfo); // receives PROCESS_INFORMATION if (!processStarted) { - throw Error("BlackBoxExec", windows_error("Unable to start program `" + - program + "'", GetLastError())); + throw Error("BlackBoxExec", + windows_error("starting blackbox process failed for program `" + + program + "'", GetLastError())); } WindowsHandle process_handle(piProcInfo.hProcess); WindowsHandle thread_handle(piProcInfo.hThread); @@ -1576,7 +1581,7 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { ) { std::vector int_in(int_input.size()); std::vector int_out(int_output.size()); - for (int i = 0; i < int_in.size(); i++) { + for (size_t i = 0; i < int_in.size(); i++) { int_in[i] = static_cast(int_input[i].val()); } std::vector float_in; @@ -1596,7 +1601,7 @@ ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { return ES_FAILED; } - for (int i = 0; i < int_out.size(); i++) { + for (size_t i = 0; i < int_out.size(); i++) { GECODE_ME_CHECK(int_output[i].eq(home, static_cast(int_out[i]))); } #ifdef GECODE_HAS_FLOAT_VARS diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index d4351dd1a9..1f40ebf7c5 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -31,8 +31,8 @@ * */ -#ifndef __FLATZINC_BLACKBOX_HH__ -#define __FLATZINC_BLACKBOX_HH__ +#ifndef GECODE_FLATZINC_BLACKBOX_HH +#define GECODE_FLATZINC_BLACKBOX_HH #include #include @@ -74,6 +74,12 @@ public: std::vector &float_out) = 0; }; +/// Access to FlatZincSpace state used only while posting blackbox constraints +class BlackBoxAccess { +public: + static SharedHandle& state(FlatZincSpace& s); +}; + /// Implementation of a black box function that dynamically loads a library and /// run a contained function. /// @@ -238,7 +244,7 @@ public: home.notice(*this, AP_DISPOSE); } /// Cost function (defined as exponential) - PropCost cost(const Space &home, const ModEventDelta &med) const override { + PropCost cost(const Space &, const ModEventDelta &) const override { return PropCost::crazy(PropCost::HI, int_input.size() #ifdef GECODE_HAS_FLOAT_VARS + float_input.size() @@ -403,7 +409,7 @@ public: home.notice(*this, AP_WEAKLY); } /// Cost function (defined as exponential) - PropCost cost(const Space &home, const ModEventDelta &med) const override { + PropCost cost(const Space &, const ModEventDelta &) const override { return PropCost::crazy(PropCost::HI, ivar.size() #ifdef GECODE_HAS_FLOAT_VARS + fvar.size() @@ -528,4 +534,4 @@ void blackbox_bounds(Home home, SharedHandle &black_box_state, } // namespace FlatZinc } // namespace Gecode -#endif //__FLATZINC_BLACKBOX_HH__ +#endif // GECODE_FLATZINC_BLACKBOX_HH diff --git a/gecode/flatzinc/flatzinc.cpp b/gecode/flatzinc/flatzinc.cpp index 090cfebc59..43172b1152 100644 --- a/gecode/flatzinc/flatzinc.cpp +++ b/gecode/flatzinc/flatzinc.cpp @@ -869,9 +869,9 @@ namespace Gecode { namespace FlatZinc { } SharedHandle& - FlatZincSpace::blackBoxState(void) { - assert(_initData != nullptr); - return _initData->blackBoxState; + BlackBoxAccess::state(FlatZincSpace& s) { + assert(s._initData != nullptr); + return s._initData->blackBoxState; } void @@ -1861,9 +1861,11 @@ namespace Gecode { namespace FlatZinc { const FlatZincOptions& opt, Support::Timer& t_total) { #ifdef GECODE_HAS_GIST if (opt.mode() == SM_GIST) { + BlackBoxStateHandle black_box_state(BlackBoxAccess::state(*this)); + (void) status(); + black_box_state.rethrow(); FZPrintingInspector pi(p); FZPrintingComparator pc(p); - BlackBoxStateHandle black_box_state(blackBoxState()); (void) GistEngine >::explore(this,opt,&pi,&pc); black_box_state.rethrow(); return; @@ -1876,7 +1878,7 @@ namespace Gecode { namespace FlatZinc { if (status(sstat) != SS_FAILED) { n_p = PropagatorGroup::all.size(*this); } - BlackBoxStateHandle black_box_state(blackBoxState()); + BlackBoxStateHandle black_box_state(BlackBoxAccess::state(*this)); black_box_state.rethrow(); Search::Options o; o.stop = Driver::CombinedStop::create(opt.node(), opt.fail(), opt.time(), diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn index 3f95dbebde..e929095667 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn @@ -42,6 +42,9 @@ % must keep reading requests and writing complete newline-terminated responses; % otherwise it can block the solver. During parallel search each worker thread % gets its own process. +% POSIX exec teardown contains descendants that remain in the spawned process +% group; helpers that deliberately detach are outside this containment and +% trusted-code contract. % % All blackboxes must be deterministic from FlatZinc's point of view: equal % input arrays must produce equal output arrays, regardless of call order or diff --git a/gecode/flatzinc/registry.cpp b/gecode/flatzinc/registry.cpp index 29162dbab8..0d7bbaf6db 100755 --- a/gecode/flatzinc/registry.cpp +++ b/gecode/flatzinc/registry.cpp @@ -1736,7 +1736,7 @@ namespace Gecode { namespace FlatZinc { "Blackbox propagator cannot use floating point values when Gecode is compiled without floating point decision variable support."); } #endif - FlatZinc::blackbox(s, s.blackBoxState(), int_input, int_output, + FlatZinc::blackbox(s, BlackBoxAccess::state(s), int_input, int_output, #ifdef GECODE_HAS_FLOAT_VARS float_input, float_output, #endif @@ -1762,7 +1762,7 @@ float_input, float_output, for (int i = 0; i < flat_reason.size(); i++) { reason[i] = flat_reason[i]; } - FlatZinc::blackbox_bounds(s, s.blackBoxState(), ivar, + FlatZinc::blackbox_bounds(s, BlackBoxAccess::state(s), ivar, #ifdef GECODE_HAS_FLOAT_VARS fvar, #endif diff --git a/test/flatzinc.cpp b/test/flatzinc.cpp index 34a7cc48f3..7c6d6724a4 100755 --- a/test/flatzinc.cpp +++ b/test/flatzinc.cpp @@ -33,6 +33,8 @@ #include "test/flatzinc.hh" +#include + namespace Test { namespace FlatZinc { namespace { @@ -66,14 +68,19 @@ namespace Test { namespace FlatZinc { } FlatZincTest::FlatZincTest(const std::string& name, const std::string& source, - const std::string& expected, bool allSolutions, std::vector cmdlineOpt) + const std::string& expected, bool allSolutions, + std::vector cmdlineOpt, + OutputCheck check, BeforeRun before) : Base("FlatZinc::"+name), _name(name), _source(source), _expected(expected), - _allSolutions(allSolutions), _cmdlineOpt(cmdlineOpt) {} + _allSolutions(allSolutions), _cmdlineOpt(cmdlineOpt), + _check(check), _before(before) {} FlatZincErrorTest::FlatZincErrorTest(const std::string& name, const std::string& source, - std::vector cmdlineOpt) - : FlatZincTest(name, source, "", false, cmdlineOpt) {} + std::vector cmdlineOpt, + std::string expectedMessage) + : FlatZincTest(name, source, "", false, cmdlineOpt), + _expectedMessage(expectedMessage) {} bool FlatZincTest::run(void) { @@ -93,10 +100,13 @@ namespace Test { namespace FlatZinc { } fznopt.allSolutions(_allSolutions); Gecode::FlatZinc::Printer p; - Gecode::FlatZinc::FlatZincSpace* fg = nullptr; try { + if (_before) { + _before(); + } std::stringstream ss(_source); - fg = Gecode::FlatZinc::parse(ss, p, olog); + std::unique_ptr fg( + Gecode::FlatZinc::parse(ss, p, olog)); if (fg) { fg->createBranchers(p, fg->solveAnnotations(), fznopt, @@ -105,19 +115,19 @@ namespace Test { namespace FlatZinc { std::ostringstream os; fg->run(os, p, fznopt, t_total); - if (_expected == os.str()) { + const std::string output = os.str(); + fg.reset(); + if (_check ? _check(output) : (_expected == output)) { return true; - } else { - if (opt.log) - olog << "FlatZinc produced the following output:\n" << os.str() << "\n"; - return false; } + if (opt.log) + olog << "FlatZinc produced the following output:\n" << output << "\n"; + return false; } else { if (opt.log) olog << "Could not parse input\n"; return false; } - delete fg; } catch (Gecode::FlatZinc::Error& e) { if (opt.log) olog << ind(2) << "FlatZinc error : " << e.toString() << std::endl; @@ -143,31 +153,34 @@ namespace Test { namespace FlatZinc { fznopt.parse(argc, argv.data()); } Gecode::FlatZinc::Printer p; - Gecode::FlatZinc::FlatZincSpace* fg = nullptr; + std::ostringstream os; try { std::stringstream ss(_source); - fg = Gecode::FlatZinc::parse(ss, p, olog); + std::unique_ptr fg( + Gecode::FlatZinc::parse(ss, p, olog)); if (fg) { fg->createBranchers(p, fg->solveAnnotations(), fznopt, false, olog); fg->shrinkArrays(p); - std::ostringstream os; fg->run(os, p, fznopt, t_total); } - delete fg; return false; } catch (Gecode::FlatZinc::Error& e) { - delete fg; + const std::string message = e.toString(); if (opt.log) olog << ind(2) << "Expected FlatZinc error : " - << e.toString() << std::endl; - return true; + << message << std::endl; + return (os.str().find("----------") == std::string::npos) && + (_expectedMessage.empty() || + (message.find(_expectedMessage) != std::string::npos)); } catch (Gecode::Exception& e) { - delete fg; + const std::string message = e.what(); if (opt.log) olog << ind(2) << "Expected Gecode exception : " - << e.what() << std::endl; - return true; + << message << std::endl; + return (os.str().find("----------") == std::string::npos) && + (_expectedMessage.empty() || + (message.find(_expectedMessage) != std::string::npos)); } return false; } diff --git a/test/flatzinc.hh b/test/flatzinc.hh index db51a1b4e3..4295a007e8 100644 --- a/test/flatzinc.hh +++ b/test/flatzinc.hh @@ -39,6 +39,7 @@ #include "test/test.hh" +#include #include #include @@ -53,25 +54,34 @@ namespace Test { */ class FlatZincTest : public Base { protected: + typedef std::function OutputCheck; + typedef std::function BeforeRun; std::string _name; std::string _source; std::string _expected; bool _allSolutions; std::vector _cmdlineOpt; + OutputCheck _check; + BeforeRun _before; public: /// Construct and register test FlatZincTest(const std::string& name, const std::string& source, const std::string& expected, bool allSolutions = false, - std::vector cmdlineOpt = {}); + std::vector cmdlineOpt = {}, + OutputCheck check = OutputCheck(), + BeforeRun before = BeforeRun()); /// Perform test virtual bool run(void); }; class FlatZincErrorTest : public FlatZincTest { + private: + std::string _expectedMessage; public: /// Construct and register test FlatZincErrorTest(const std::string& name, const std::string& source, - std::vector cmdlineOpt = {}); + std::vector cmdlineOpt = {}, + std::string expectedMessage = ""); /// Perform test virtual bool run(void); }; diff --git a/test/flatzinc/blackbox-dll.cpp b/test/flatzinc/blackbox-dll.cpp new file mode 100644 index 0000000000..74ef4a770f --- /dev/null +++ b/test/flatzinc/blackbox-dll.cpp @@ -0,0 +1,194 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Copyright: + * Jip J. Dekker, 2026 + * + * This file is part of Gecode, the generic constraint + * development environment: + * http://www.gecode.dev + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#define GECODE_BLACKBOX_EXPORT __declspec(dllexport) +#define GECODE_BLACKBOX_CALL __stdcall +#else +#include +#include +#include +#if defined(__GNUC__) || defined(__clang__) +#define GECODE_BLACKBOX_EXPORT __attribute__((visibility("default"))) +#else +#define GECODE_BLACKBOX_EXPORT +#endif +#define GECODE_BLACKBOX_CALL +#endif + +namespace { + + enum class Mode { normal, nan }; + + struct Instance { + const Mode mode; + const std::string log; + const unsigned int id; + + Instance(const char** args, size_t n_args, unsigned int id0) + : mode((n_args > 0) && (std::strcmp(args[0], "nan") == 0) + ? Mode::nan : Mode::normal), + log((n_args > 1) ? args[1] : ""), id(id0) {} + Instance(const Instance& other, unsigned int id0) + : mode(other.mode), log(other.log), id(id0) {} + }; + + std::atomic next_id(0); + + unsigned int + instance_id(void) { + return next_id.fetch_add(1, std::memory_order_relaxed) + 1; + } + + void + record(const std::string& log, const std::string& event, unsigned int id) { + if (log.empty()) { + return; + } + const std::string line = event + " " + std::to_string(id) + "\n"; +#ifdef _WIN32 + HANDLE file = CreateFileA(log.c_str(), FILE_APPEND_DATA, + FILE_SHARE_READ | FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + return; + } + DWORD written; + (void)WriteFile(file, line.data(), static_cast(line.size()), + &written, nullptr); + (void)CloseHandle(file); +#else + int fd; + do { + fd = open(log.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0600); + } while ((fd == -1) && (errno == EINTR)); + if (fd == -1) { + return; + } + ssize_t n; + do { + n = write(fd, line.data(), line.size()); + } while ((n == -1) && (errno == EINTR)); + (void)close(fd); +#endif + } + + bool + has_two_starts(const std::string& log) { + std::ifstream in(log.c_str()); + std::set ids; + std::string event; + unsigned int id; + while (in >> event >> id) { + if (event == "start") { + ids.insert(id); + } + } + return ids.size() >= 2; + } + + void + wait_for_peer(const std::string& log, unsigned int id) { + for (unsigned int i = 0; i < 500; ++i) { + if (has_two_starts(log)) { + return; + } +#ifdef _WIN32 + Sleep(10); +#else + usleep(10000); +#endif + } + record(log, "timeout", id); + } + +} + +extern "C" GECODE_BLACKBOX_EXPORT void* GECODE_BLACKBOX_CALL +fzn_init(const char** args, size_t n_args) { + Instance* instance = new Instance(args, n_args, instance_id()); + record(instance->log, "init", instance->id); + return instance; +} + +extern "C" GECODE_BLACKBOX_EXPORT void* GECODE_BLACKBOX_CALL +fzn_clone(void* value) { + Instance* instance = new Instance(*static_cast(value), + instance_id()); + record(instance->log, "clone", instance->id); + return instance; +} + +extern "C" GECODE_BLACKBOX_EXPORT void GECODE_BLACKBOX_CALL +fzn_free(void* value) { + Instance* instance = static_cast(value); + record(instance->log, "free", instance->id); + delete instance; +} + +extern "C" GECODE_BLACKBOX_EXPORT void GECODE_BLACKBOX_CALL +fzn_blackbox(void* value, const int64_t* int_input, size_t n_int_input, + const double* float_input, size_t n_float_input, + int64_t* int_output, size_t n_int_output, + double* float_output, size_t n_float_output) { + const Instance* instance = static_cast(value); + if (!instance->log.empty() && (n_int_input > 0)) { + record(instance->log, "start", instance->id); + wait_for_peer(instance->log, instance->id); + record(instance->log, "ready", instance->id); + } + for (size_t i = 0; i < n_int_output; ++i) { + int_output[i] = (n_int_input == 0) + ? 1 + : int_input[i % n_int_input]; + } + for (size_t i = 0; i < n_float_output; ++i) { + if (instance->mode == Mode::nan) { + const uint64_t nan = UINT64_C(0x7ff8000000000000); + std::memcpy(&float_output[i], &nan, sizeof(nan)); + } else { + float_output[i] = (n_float_input == 0) + ? 0.0 : float_input[i % n_float_input]; + } + } +} + +// STATISTICS: test-flatzinc diff --git a/test/flatzinc/blackbox-exec.cpp b/test/flatzinc/blackbox-exec.cpp new file mode 100644 index 0000000000..75d4d355e8 --- /dev/null +++ b/test/flatzinc/blackbox-exec.cpp @@ -0,0 +1,268 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Copyright: + * Jip J. Dekker, 2026 + * + * This file is part of Gecode, the generic constraint + * development environment: + * http://www.gecode.dev + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#include +#include +#endif + +namespace { + + std::string + process_id(void) { +#ifdef _WIN32 + return std::to_string(static_cast(GetCurrentProcessId())); +#else + return std::to_string(static_cast(getpid())); +#endif + } + + void + record(const std::string& log, const std::string& event, + const std::string& id) { + if (log.empty()) { + return; + } + const std::string line = event + " " + id + "\n"; +#ifdef _WIN32 + HANDLE file = CreateFileA(log.c_str(), FILE_APPEND_DATA, + FILE_SHARE_READ | FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + return; + } + DWORD written; + (void)WriteFile(file, line.data(), static_cast(line.size()), + &written, nullptr); + (void)CloseHandle(file); +#else + int fd; + do { + fd = open(log.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0600); + } while ((fd == -1) && (errno == EINTR)); + if (fd == -1) { + return; + } + ssize_t n; + do { + n = write(fd, line.data(), line.size()); + } while ((n == -1) && (errno == EINTR)); + (void)close(fd); +#endif + } + + bool + has_two_starts(const std::string& log) { + std::ifstream in(log.c_str()); + std::set ids; + std::string event; + std::string id; + while (in >> event >> id) { + if (event == "start") { + ids.insert(id); + } + } + return ids.size() >= 2; + } + + void + wait_for_peer(const std::string& log, const std::string& id) { + for (unsigned int i = 0; i < 500; ++i) { + if (has_two_starts(log)) { + return; + } +#ifdef _WIN32 + Sleep(10); +#else + usleep(10000); +#endif + } + record(log, "timeout", id); + } + + bool + first_integer(const std::string& request, int64_t& value) { + const std::string::size_type end = request.find(';'); + if ((end == std::string::npos) || (end == 0)) { + return false; + } + const std::string integer = request.substr(0, end); + char* last = nullptr; + errno = 0; + const long long parsed = std::strtoll(integer.c_str(), &last, 10); + if ((last == integer.c_str()) || (*last != '\0') || (errno == ERANGE) || + (parsed < (std::numeric_limits::min)()) || + (parsed > (std::numeric_limits::max)())) { + return false; + } + value = static_cast(parsed); + return true; + } + + void + write_response(const std::string& value) { + std::cout.write(value.data(), static_cast(value.size())); + std::cout << std::endl; + std::cout.flush(); + } + + const std::string* + fixed_response(const std::string& mode) { + static const std::string value7("7;"); + static const std::string bounds2("5,5;"); + static const std::string bounds4("5,5,5,5;"); + static const std::string mixed("5,5;5,5"); + static const std::string nan(";nan"); + static const std::string nul(";\0x", 3); + static const std::string malformed("x;"); + if (mode == "value7") { + return &value7; + } else if (mode == "bounds2") { + return &bounds2; + } else if (mode == "bounds4") { + return &bounds4; + } else if (mode == "mixed") { + return &mixed; + } else if (mode == "nan") { + return &nan; + } else if (mode == "nul") { + return &nul; + } else if (mode == "malformed") { + return &malformed; + } + return nullptr; + } + + void + fault(int64_t kind) { + if (kind == 1) { + write_response("invalid"); + } else if (kind == 2) { + return; + } else if (kind == 3) { + write_response(std::string(";\0x", 3)); + } else if (kind == 5) { + write_response(std::to_string((std::numeric_limits::max)()) + + ";"); + } else { + write_response(std::string(1024U * 1024U + 1U, 'x')); + } + } + +} + +int +main(int argc, char* argv[]) { + const std::string mode = (argc > 1) ? argv[1] : "normal"; + const std::string log = (argc > 2) ? argv[2] : ""; + unsigned int round = 0; +#ifndef _WIN32 + bool descendant_started = false; +#endif + std::string request; + while (std::getline(std::cin, request)) { + ++round; + int64_t value = 0; + const bool has_value = first_integer(request, value); + if (mode == "fault") { + fault(has_value ? value : 1); + return 0; + } + if (mode == "descendant") { +#ifndef _WIN32 + if (!descendant_started) { + int ready[2]; + if (pipe(ready) != 0) { + return 1; + } + const pid_t descendant = fork(); + if (descendant == 0) { + (void)close(ready[0]); + record(log, "descendant", process_id()); + const char started = '1'; + (void)write(ready[1], &started, 1); + (void)close(ready[1]); + for (;;) { + pause(); + } + } + (void)close(ready[1]); + if (descendant <= 0) { + (void)close(ready[0]); + return 1; + } + char started = '\0'; + ssize_t n; + do { + n = read(ready[0], &started, 1); + } while ((n == -1) && (errno == EINTR)); + (void)close(ready[0]); + if ((n != 1) || (started != '1')) { + return 1; + } + descendant_started = true; + } +#endif + write_response("1;"); + continue; + } + + if (const std::string* value = fixed_response(mode)) { + write_response(*value); + continue; + } + + if (!log.empty() && has_value) { + const std::string id = process_id(); + record(log, "start", id); + wait_for_peer(log, id); + record(log, "ready", id); + } + write_response(std::to_string(has_value ? value : + static_cast(round)) + ";"); + } + return 0; +} + +// STATISTICS: test-flatzinc diff --git a/test/flatzinc/blackbox.cpp b/test/flatzinc/blackbox.cpp index 52a85414c2..caa51f54bf 100644 --- a/test/flatzinc/blackbox.cpp +++ b/test/flatzinc/blackbox.cpp @@ -33,7 +33,24 @@ #include "test/flatzinc.hh" -#ifdef GECODE_HAS_FLOAT_VARS +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifdef GECODE_HAS_THREADS +#include +#endif + +#if defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) +#include +#include +#endif namespace Test { namespace FlatZinc { @@ -51,146 +68,449 @@ namespace Test { namespace FlatZinc { "array[int] of var float: float_input, " "array[int] of int: flat_reason);\n"; - const char* python_value_7 = - "blackbox_exec(\"python3\", " - "[\"-u\", \"-c\", " - "\"import sys; [print(chr(55)+chr(59), flush=True) " - "for line in sys.stdin]\"])"; - - const char* python_bounds_5 = - "blackbox_exec(\"python3\", " - "[\"-u\", \"-c\", " - "\"import sys; [print(chr(53)+chr(44)+chr(53)+chr(59), " - "flush=True) for line in sys.stdin]\"])"; - - const char* python_bounds_5_twice = - "blackbox_exec(\"python3\", " - "[\"-u\", \"-c\", " - "\"import sys; [print(chr(53)+chr(44)+chr(53)+chr(44)+chr(53)+" - "chr(44)+chr(53)+chr(59), flush=True) for line in sys.stdin]\"])"; - - const char* python_mixed_bounds_5 = - "blackbox_exec(\"python3\", " - "[\"-u\", \"-c\", " - "\"import sys; [print(chr(53)+chr(44)+chr(53)+chr(59)+chr(53)+" - "chr(44)+chr(53), flush=True) for line in sys.stdin]\"])"; - - const char* python_nan = - "blackbox_exec(\"python3\", " - "[\"-u\", \"-c\", " - "\"import sys; [print(chr(59)+chr(110)+chr(97)+chr(110), " - "flush=True) for line in sys.stdin]\"])"; - - const char* python_nul = - "blackbox_exec(\"python3\", " - "[\"-u\", \"-c\", " - "\"import sys; [print(chr(59)+chr(0)+chr(120), " - "flush=True) for line in sys.stdin]\"])"; - - const char* python_malformed = - "blackbox_exec(\"python3\", " - "[\"-u\", \"-c\", " - "\"import sys; [print(chr(120)+chr(59), " - "flush=True) for line in sys.stdin]\"])"; + std::string + fzn_string(const std::string& value) { + std::string quoted("\""); + for (char c : value) { + if ((c == '\\') || (c == '\"')) { + quoted += '\\'; + } + quoted += c; + } + return quoted + "\""; + } + + std::string + fixture_annotation(const char* mode, const std::string& fixture, + const std::vector& args) { + std::string annotation("blackbox_"); + annotation += mode; + annotation += "(" + fzn_string(fixture); + if (!args.empty()) { + annotation += ", ["; + for (size_t i = 0; i < args.size(); ++i) { + if (i != 0) { + annotation += ", "; + } + annotation += fzn_string(args[i]); + } + annotation += "]"; + } + return annotation + ")"; + } + + std::string + fixture_log(const std::string& name) { + const char* base = std::getenv("GECODE_TEST_BLACKBOX_LOG"); + return (base == nullptr) ? std::string() : std::string(base) + "." + name; + } + + void + reset_log(const std::string& log) { + std::remove(log.c_str()); + } + + std::vector > + read_log(const std::string& log) { + std::vector > entries; + std::ifstream in(log.c_str()); + std::string event; + std::string id; + while (in >> event >> id) { + entries.push_back(std::make_pair(event, id)); + } + return entries; + } + + bool + concurrent_log(const std::string& log) { + const std::vector > entries = + read_log(log); + std::set starts; + std::set ready; + if ((entries.size() != 4) || (entries[0].first != "start") || + (entries[1].first != "start") || (entries[2].first != "ready") || + (entries[3].first != "ready")) { + return false; + } + for (const auto& entry : entries) { + (entry.first == "start" ? starts : ready).insert(entry.second); + } + return (starts.size() == 2) && (ready == starts); + } + + bool + dll_parallel_lifecycle(const std::string& log) { + const std::vector > entries = + read_log(log); + std::set clones; + std::set starts; + std::set ready; + std::set freed; + std::vector calls; + std::string prototype; + unsigned int inits = 0; + unsigned int clone_count = 0; + unsigned int start_count = 0; + unsigned int ready_count = 0; + unsigned int free_count = 0; + bool frees_after_calls = true; + for (const auto& entry : entries) { + if (entry.first == "init") { + ++inits; + prototype = entry.second; + } else if (entry.first == "clone") { + ++clone_count; + clones.insert(entry.second); + } else if (entry.first == "start") { + ++start_count; + starts.insert(entry.second); + calls.push_back(entry.first); + } else if (entry.first == "ready") { + ++ready_count; + ready.insert(entry.second); + calls.push_back(entry.first); + } else if (entry.first == "free") { + ++free_count; + freed.insert(entry.second); + frees_after_calls = frees_after_calls && (calls.size() == 4); + } else { + return false; + } + } + std::set instances(clones); + instances.insert(prototype); + return (entries.size() == 10) && (entries[0].first == "init") && + (inits == 1) && (clone_count == 2) && (start_count == 2) && + (ready_count == 2) && (free_count == 3) && (clones.size() == 2) && + (clones.find(prototype) == clones.end()) && (starts == clones) && + (ready == clones) && (freed == instances) && frees_after_calls && + (calls == std::vector{"start", "start", "ready", "ready"}); + } + + bool + dll_model_lifecycle(const std::string& log) { + const std::vector > entries = + read_log(log); +#ifdef GECODE_HAS_THREADS + if (entries.size() != 8) { + return false; + } + for (size_t i = 0; i < entries.size(); i += 4) { + if ((entries[i].first != "init") || + (entries[i+1].first != "clone") || + (entries[i+2] != std::make_pair(std::string("free"), + entries[i+1].second)) || + (entries[i+3] != std::make_pair(std::string("free"), + entries[i].second))) { + return false; + } + } +#else + if (entries.size() != 4) { + return false; + } + for (size_t i = 0; i < entries.size(); i += 2) { + if ((entries[i].first != "init") || + (entries[i+1] != std::make_pair(std::string("free"), + entries[i].second))) { + return false; + } + } +#endif + return true; + } + +#if defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) + bool + descendant_gone(const std::string& output, const std::string& log) { + const std::vector > entries = + read_log(log); + reset_log(log); + if (output.find("y = 1;") == std::string::npos) { + return false; + } + for (const auto& entry : entries) { + if (entry.first == "descendant") { + const long pid = std::strtol(entry.second.c_str(), nullptr, 10); + if (pid <= 0) { + return false; + } + for (unsigned int i = 0; i < 500; ++i) { + if ((kill(static_cast(pid), 0) == -1) && (errno == ESRCH)) { + return true; + } + usleep(10000); + } + return false; + } + } + return false; + } +#endif } namespace Blackbox { +#ifdef GECODE_HAS_THREADS + template + bool + parallel_runs(T& black_box) { + bool echoed[2] = {false, false}; + std::exception_ptr exception[2]; + auto run = [&black_box, &echoed, &exception](int i) { + try { + const int64_t input = (i == 0) ? -1 : i + 1; + std::vector int_out(1); + std::vector float_out; + black_box.run({input}, {}, int_out, float_out); + echoed[i] = (int_out[0] == input); + } catch (...) { + exception[i] = std::current_exception(); + } + }; + std::thread first; + std::thread second; + auto join = [&first, &second](void) { + if (first.joinable()) { + first.join(); + } + if (second.joinable()) { + second.join(); + } + }; + try { + first = std::thread(run, 0); + second = std::thread(run, 1); + } catch (...) { + join(); + return false; + } + join(); + return (exception[0] == nullptr) && (exception[1] == nullptr) && + echoed[0] && echoed[1]; + } + +#if defined(_WIN32) || defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) + class NativeExecParallelSessions : public Base { + private: + std::string executable; + std::string log; + public: + NativeExecParallelSessions(const std::string& executable0, + const std::string& log0) + : Base("FlatZinc::blackbox::native_exec_parallel_sessions"), + executable(executable0), log(log0) {} + virtual bool run(void) { + reset_log(log); + bool ok = false; + try { + Gecode::FlatZinc::BlackBoxExec black_box(executable, + {"normal", log}); + ok = parallel_runs(black_box) && concurrent_log(log); + } catch (...) {} + reset_log(log); + return ok; + } + }; +#endif + + class NativeDllParallelCalls : public Base { + private: + std::string library; + std::string log; + public: + NativeDllParallelCalls(const std::string& library0, + const std::string& log0) + : Base("FlatZinc::blackbox::native_dll_parallel_calls"), + library(library0), log(log0) {} + virtual bool run(void) { + reset_log(log); + bool ok = false; + try { + { + Gecode::FlatZinc::BlackBoxLibrary black_box(library, + {"normal", log}); + ok = parallel_runs(black_box); + } + ok = ok && dll_parallel_lifecycle(log); + } catch (...) {} + reset_log(log); + return ok; + } + }; +#endif + class Create { public: /// Perform creation and registration Create(void) { - (void) new FlatZincTest("blackbox::constant_value", - std::string(blackbox_decl) + - "var 7..7: y :: output_var;\n" - "constraint gecode_blackbox([], [], [y], []) :: " + - python_value_7 + - ";\n" - "solve satisfy;\n", - "y = 7;\n----------\n"); - - (void) new FlatZincTest("blackbox::constant_value_unsat", - std::string(blackbox_decl) + - "var 8..8: y;\n" - "constraint gecode_blackbox([], [], [y], []) :: " + - python_value_7 + - ";\n" - "solve satisfy;\n", - "=====UNSATISFIABLE=====\n"); - - (void) new FlatZincTest("blackbox::reason_independent_bounds", - std::string(blackbox_bounds_decl) + - "var 5..5: x :: output_var;\n" - "constraint gecode_blackbox_bounds([x], [], [1,0,0]) :: " + - python_bounds_5 + - ";\n" - "solve satisfy;\n", - "x = 5;\n----------\n"); - - (void) new FlatZincTest("blackbox::reason_independent_bounds_unsat", - std::string(blackbox_bounds_decl) + - "var 6..6: x;\n" - "constraint gecode_blackbox_bounds([x], [], [1,0,0]) :: " + - python_bounds_5 + - ";\n" - "solve satisfy;\n", - "=====UNSATISFIABLE=====\n"); - - (void) new FlatZincTest("blackbox::reason_dependent_bounds", - std::string(blackbox_bounds_decl) + - "var 5..5: x :: output_var;\n" - "constraint gecode_blackbox_bounds([x], [], [1,1,1,1,0]) :: " + - python_bounds_5 + - ";\n" - "solve satisfy;\n", - "x = 5;\n----------\n"); - - (void) new FlatZincErrorTest("blackbox::missing_bounds_reason_entry", - std::string(blackbox_bounds_decl) + - "var 0..10: x;\n" - "var 0.0..10.0: y;\n" - "constraint gecode_blackbox_bounds([x], [y], [1,0,0]) :: " + - python_mixed_bounds_5 + - ";\n" - "solve satisfy;\n"); - - (void) new FlatZincErrorTest("blackbox::duplicate_bounds_reason_entry", - std::string(blackbox_bounds_decl) + - "var 0..10: x;\n" - "var 0..10: y;\n" - "constraint gecode_blackbox_bounds([x,y], [], [1,0,0,1,0,0]) :: " + - python_bounds_5_twice + - ";\n" - "solve satisfy;\n"); - - (void) new FlatZincErrorTest("blackbox::invalid_bounds_reason_code", - std::string(blackbox_bounds_decl) + - "var 0..10: x;\n" - "constraint gecode_blackbox_bounds([x], [], [1,1,1,0,0]) :: " + - python_bounds_5 + - ";\n" - "solve satisfy;\n"); - - (void) new FlatZincErrorTest("blackbox::invalid_float_output", - std::string(blackbox_decl) + - "var 0.0..10.0: y;\n" - "constraint gecode_blackbox([], [], [], [y]) :: " + - python_nan + - ";\n" - "solve satisfy;\n"); - - (void) new FlatZincErrorTest("blackbox::nul_output", - std::string(blackbox_decl) + - "constraint gecode_blackbox([], [], [], []) :: " + - python_nul + - ";\n" - "solve satisfy;\n"); - (void) new FlatZincErrorTest("blackbox::malformed_annotation", std::string(blackbox_decl) + "var 0..1: y;\n" "constraint gecode_blackbox([], [], [y], []) :: " "blackbox_exec([]);\n" - "solve satisfy;\n"); + "solve satisfy;\n", {}, + "expected a target string and an argument array"); + +#if defined(_WIN32) || defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) + const char* exec = std::getenv("GECODE_TEST_BLACKBOX_EXEC"); + if (exec != nullptr) { + const std::string executable(exec); + (void) new FlatZincTest("blackbox::constant_value", + std::string(blackbox_decl) + + "var 7..7: y :: output_var;\n" + "constraint gecode_blackbox([], [], [y], []) :: " + + fixture_annotation("exec", executable, {"value7"}) + ";\n" + "solve satisfy;\n", + "y = 7;\n----------\n"); + + (void) new FlatZincTest("blackbox::constant_value_unsat", + std::string(blackbox_decl) + + "var 8..8: y;\n" + "constraint gecode_blackbox([], [], [y], []) :: " + + fixture_annotation("exec", executable, {"value7"}) + ";\n" + "solve satisfy;\n", + "=====UNSATISFIABLE=====\n"); + + (void) new FlatZincTest("blackbox::reason_independent_bounds", + std::string(blackbox_bounds_decl) + + "var 5..5: x :: output_var;\n" + "constraint gecode_blackbox_bounds([x], [], [1,0,0]) :: " + + fixture_annotation("exec", executable, {"bounds2"}) + ";\n" + "solve satisfy;\n", + "x = 5;\n----------\n"); + + (void) new FlatZincTest("blackbox::reason_independent_bounds_unsat", + std::string(blackbox_bounds_decl) + + "var 6..6: x;\n" + "constraint gecode_blackbox_bounds([x], [], [1,0,0]) :: " + + fixture_annotation("exec", executable, {"bounds2"}) + ";\n" + "solve satisfy;\n", + "=====UNSATISFIABLE=====\n"); + + (void) new FlatZincTest("blackbox::reason_dependent_bounds", + std::string(blackbox_bounds_decl) + + "var 5..5: x :: output_var;\n" + "constraint gecode_blackbox_bounds([x], [], [1,1,1,1,0]) :: " + + fixture_annotation("exec", executable, {"bounds2"}) + ";\n" + "solve satisfy;\n", + "x = 5;\n----------\n"); + +#ifdef GECODE_HAS_FLOAT_VARS + (void) new FlatZincErrorTest("blackbox::missing_bounds_reason_entry", + std::string(blackbox_bounds_decl) + + "var 0..10: x;\n" + "var 0.0..10.0: y;\n" + "constraint gecode_blackbox_bounds([x], [y], [1,0,0]) :: " + + fixture_annotation("exec", executable, {"mixed"}) + ";\n" + "solve satisfy;\n", {}, "missing explained variable entry"); +#endif + + (void) new FlatZincErrorTest("blackbox::duplicate_bounds_reason_entry", + std::string(blackbox_bounds_decl) + + "var 0..10: x;\n" + "var 0..10: y;\n" + "constraint gecode_blackbox_bounds([x,y], [], [1,0,0,1,0,0]) :: " + + fixture_annotation("exec", executable, {"bounds4"}) + ";\n" + "solve satisfy;\n", {}, "duplicate explained variable index"); + + (void) new FlatZincErrorTest("blackbox::invalid_bounds_reason_code", + std::string(blackbox_bounds_decl) + + "var 0..10: x;\n" + "constraint gecode_blackbox_bounds([x], [], [1,1,1,0,0]) :: " + + fixture_annotation("exec", executable, {"bounds2"}) + ";\n" + "solve satisfy;\n", {}, "dependency bound code is out of range"); + +#ifdef GECODE_HAS_FLOAT_VARS + (void) new FlatZincErrorTest("blackbox::invalid_float_output", + std::string(blackbox_decl) + + "var 0.0..10.0: y;\n" + "constraint gecode_blackbox([], [], [], [y]) :: " + + fixture_annotation("exec", executable, {"nan"}) + ";\n" + "solve satisfy;\n", {}, "Failed to read output float 0"); +#endif + + (void) new FlatZincErrorTest("blackbox::nul_output", + std::string(blackbox_decl) + + "constraint gecode_blackbox([], [], [], []) :: " + + fixture_annotation("exec", executable, {"nul"}) + ";\n" + "solve satisfy;\n", {}, "response contains NUL data"); + + (void) new FlatZincErrorTest("blackbox::malformed_exec_parallel", + std::string(blackbox_decl) + + "var 0..1: x :: output_var;\n" + "var 0..1: y :: output_var;\n" + "constraint gecode_blackbox([x], [], [y], []) :: " + + fixture_annotation("exec", executable, {"malformed"}) + ";\n" + "solve :: int_search([x], first_fail, indomain_min, complete) " + "satisfy;\n", + {"-p", "2"}, "Failed to read output integer 0"); + + (void) new FlatZincTest("blackbox::native_exec_rounds", + std::string(blackbox_decl) + + "var 1..1: a :: output_var;\n" + "var 2..2: b :: output_var;\n" + "constraint gecode_blackbox([], [], [a], []) :: " + + fixture_annotation("exec", executable, {"normal"}) + ";\n" + "constraint gecode_blackbox([], [], [b], []) :: " + + fixture_annotation("exec", executable, {"normal"}) + ";\n" + "solve satisfy;\n", + "a = 1;\nb = 2;\n----------\n"); + + for (int kind = 1; kind <= 5; ++kind) { + const char* expected = nullptr; + switch (kind) { + case 1: + expected = "Failed to read output integer 0"; + break; + case 2: + expected = "provided an incomplete response"; + break; + case 3: + expected = "response contains NUL data"; + break; + case 4: + expected = "response exceeds the size limit"; + break; + default: + expected = "integer 0 is outside Gecode's integer range"; + break; + } + (void) new FlatZincErrorTest( + "blackbox::native_exec_fault_" + std::to_string(kind), + std::string(blackbox_decl) + + "var " + std::to_string(kind) + ".." + std::to_string(kind) + + ": x;\nvar 0..1: y;\n" + "constraint gecode_blackbox([x], [], [y], []) :: " + + fixture_annotation("exec", executable, {"fault"}) + ";\n" + "solve satisfy;\n", {}, expected); + } + +#ifdef GECODE_HAS_THREADS + const std::string exec_log = fixture_log("exec_parallel"); + if (!exec_log.empty()) { + (void) new NativeExecParallelSessions(executable, exec_log); + } +#endif + +#if defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) + const std::string descendant_log = fixture_log("exec_descendant"); + if (!descendant_log.empty()) { + (void) new FlatZincTest("blackbox::native_exec_descendant_cleanup", + std::string(blackbox_decl) + + "var 1..1: y :: output_var;\n" + "constraint gecode_blackbox([], [], [y], []) :: " + + fixture_annotation("exec", executable, + {"descendant", descendant_log}) + ";\n" + "solve satisfy;\n", + "", false, {}, + [descendant_log](const std::string& output) { + return descendant_gone(output, descendant_log); + }, + [descendant_log](void) { reset_log(descendant_log); }); + } +#endif + } (void) new FlatZincErrorTest("blackbox::missing_exec_parallel", std::string(blackbox_decl) + @@ -200,7 +520,7 @@ namespace Test { namespace FlatZinc { "blackbox_exec(\"gecode-blackbox-missing-program\");\n" "solve :: int_search([x], first_fail, indomain_min, complete) " "satisfy;\n", - {"-p", "2"}); + {"-p", "2"}, "starting blackbox process failed"); (void) new FlatZincErrorTest("blackbox::missing_exec_root_status", std::string(blackbox_decl) + @@ -209,18 +529,50 @@ namespace Test { namespace FlatZinc { "constraint gecode_blackbox([x], [], [y], []) :: " "blackbox_exec(\"gecode-blackbox-missing-program\");\n" "solve satisfy;\n", - {"-p", "2"}); + {"-p", "2"}, "starting blackbox process failed"); +#endif - (void) new FlatZincErrorTest("blackbox::malformed_exec_parallel", - std::string(blackbox_decl) + - "var 0..1: x :: output_var;\n" - "var 0..1: y :: output_var;\n" - "constraint gecode_blackbox([x], [], [y], []) :: " + - python_malformed + - ";\n" - "solve :: int_search([x], first_fail, indomain_min, complete) " - "satisfy;\n", - {"-p", "2"}); + const char* dll = std::getenv("GECODE_TEST_BLACKBOX_DLL"); + if (dll != nullptr) { + const std::string library(dll); + const std::string dll_model_log = fixture_log("dll_model"); + if (!dll_model_log.empty()) { + (void) new FlatZincTest("blackbox::native_dll_per_constraint", + std::string(blackbox_decl) + + "var 1..1: a :: output_var;\n" + "var 1..1: b :: output_var;\n" + "constraint gecode_blackbox([], [], [a], []) :: " + + fixture_annotation("dll", library, {"normal", dll_model_log}) + + ";\nconstraint gecode_blackbox([], [], [b], []) :: " + + fixture_annotation("dll", library, {"normal", dll_model_log}) + + ";\nsolve satisfy;\n", + "a = 1;\nb = 1;\n----------\n", false, {}, + [dll_model_log](const std::string& output) { + const bool ok = (output == "a = 1;\nb = 1;\n----------\n") && + dll_model_lifecycle(dll_model_log); + reset_log(dll_model_log); + return ok; + }, + [dll_model_log](void) { reset_log(dll_model_log); }); + } + +#ifdef GECODE_HAS_THREADS + const std::string dll_log = fixture_log("dll_parallel"); + if (!dll_log.empty()) { + (void) new NativeDllParallelCalls(library, dll_log); + } +#endif + +#ifdef GECODE_HAS_FLOAT_VARS + (void) new FlatZincErrorTest("blackbox::native_dll_nonfinite", + std::string(blackbox_decl) + + "var 0.0..1.0: y;\n" + "constraint gecode_blackbox([], [], [], [y]) :: " + + fixture_annotation("dll", library, {"nan"}) + ";\n" + "solve satisfy;\n", {}, + "library output float 0 is not a finite value"); +#endif + } } }; @@ -229,6 +581,4 @@ namespace Test { namespace FlatZinc { }} -#endif - // STATISTICS: test-flatzinc From c41ca174a1025801416474adc99d05a48085a228 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sun, 12 Jul 2026 18:28:14 +0200 Subject: [PATCH 09/14] Refactor FlatZinc blackbox runtime --- CMakeLists.txt | 1 + Makefile.in | 5 +- cmake/GecodeSources.cmake | 3 +- .../{blackbox.cpp => blackbox-backend.cpp} | 442 ++--------- gecode/flatzinc/blackbox-backend.hh | 135 ++++ gecode/flatzinc/blackbox-propagator.cpp | 718 ++++++++++++++++++ gecode/flatzinc/blackbox.hh | 498 +----------- gecode/flatzinc/flatzinc.cpp | 130 ++-- gecode/flatzinc/registry.cpp | 24 +- test/flatzinc/blackbox-exec.cpp | 22 + test/flatzinc/blackbox.cpp | 51 +- 11 files changed, 1075 insertions(+), 954 deletions(-) rename gecode/flatzinc/{blackbox.cpp => blackbox-backend.cpp} (74%) create mode 100644 gecode/flatzinc/blackbox-backend.hh create mode 100644 gecode/flatzinc/blackbox-propagator.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 66e5236f47..b34a077e5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1630,6 +1630,7 @@ if(GECODE_INSTALL) PATTERN "LICENSE_1_0.txt" PATTERN "mznlib" EXCLUDE PATTERN "flatzinc/blackbox.hh" EXCLUDE + PATTERN "flatzinc/blackbox-backend.hh" EXCLUDE PATTERN "exampleplugin" EXCLUDE PATTERN "standalone-example" EXCLUDE PATTERN "abi*" EXCLUDE) diff --git a/Makefile.in b/Makefile.in index e0fa9a7dde..7c317e632b 100755 --- a/Makefile.in +++ b/Makefile.in @@ -821,12 +821,13 @@ endif # FLATZINC # -FLATZINCSRC0 = flatzinc.cpp registry.cpp branch.cpp blackbox.cpp +FLATZINCSRC0 = flatzinc.cpp registry.cpp branch.cpp blackbox-backend.cpp \ + blackbox-propagator.cpp FLATZINC_GENSRC0 = parser.tab.cpp lexer.yy.cpp FLATZINCHDR0 = ast.hh conexpr.hh option.hh parser.hh \ plugin.hh registry.hh symboltable.hh varspec.hh \ branch.hh branch.hpp lastval.hh complete.hh -FLATZINCPRIVATEHDR0 = blackbox.hh +FLATZINCPRIVATEHDR0 = blackbox.hh blackbox-backend.hh FLATZINCSRC = $(FLATZINCSRC0:%=gecode/flatzinc/%) FLATZINC_GENSRC = $(FLATZINC_GENSRC0:%=gecode/flatzinc/%) diff --git a/cmake/GecodeSources.cmake b/cmake/GecodeSources.cmake index 6a5e7cb041..8ef7efed43 100644 --- a/cmake/GecodeSources.cmake +++ b/cmake/GecodeSources.cmake @@ -232,7 +232,8 @@ set(GECODE_GIST_SOURCES ) set(GECODE_FLATZINC_SOURCES - gecode/flatzinc/blackbox.cpp + gecode/flatzinc/blackbox-backend.cpp + gecode/flatzinc/blackbox-propagator.cpp gecode/flatzinc/branch.cpp gecode/flatzinc/flatzinc.cpp gecode/flatzinc/lexer.yy.cpp diff --git a/gecode/flatzinc/blackbox.cpp b/gecode/flatzinc/blackbox-backend.cpp similarity index 74% rename from gecode/flatzinc/blackbox.cpp rename to gecode/flatzinc/blackbox-backend.cpp index 589a8bed17..961c88e546 100644 --- a/gecode/flatzinc/blackbox.cpp +++ b/gecode/flatzinc/blackbox-backend.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -92,6 +93,7 @@ extern char **environ; namespace Gecode { namespace FlatZinc { + namespace { #ifdef _WIN32 @@ -667,34 +669,35 @@ BlackBoxLibrary::instance(void) { #endif void -BlackBoxLibrary::run(const std::vector &int_in, - const std::vector &float_in, - std::vector &int_out, - std::vector &float_out) { +BlackBoxLibrary::run(BlackBoxCall& call) { #ifdef GECODE_HAS_THREADS if (library_fzn_init != nullptr) { Instance *selected = this->instance(); Support::Lock lock(selected->mutex); - library_fzn_blackbox(selected->value, int_in.data(), int_in.size(), - float_in.data(), float_in.size(), int_out.data(), - int_out.size(), float_out.data(), float_out.size()); + library_fzn_blackbox(selected->value, + call.int_input.data(), call.int_input.size(), + call.float_input.data(), call.float_input.size(), + call.int_output.data(), call.int_output.size(), + call.float_output.data(), call.float_output.size()); } else { - library_fzn_blackbox(nullptr, int_in.data(), int_in.size(), - float_in.data(), float_in.size(), int_out.data(), - int_out.size(), - float_out.data(), float_out.size()); + library_fzn_blackbox(nullptr, + call.int_input.data(), call.int_input.size(), + call.float_input.data(), call.float_input.size(), + call.int_output.data(), call.int_output.size(), + call.float_output.data(), call.float_output.size()); } #else - library_fzn_blackbox(root_instance, int_in.data(), int_in.size(), - float_in.data(), float_in.size(), int_out.data(), - int_out.size(), - float_out.data(), float_out.size()); + library_fzn_blackbox(root_instance, + call.int_input.data(), call.int_input.size(), + call.float_input.data(), call.float_input.size(), + call.int_output.data(), call.int_output.size(), + call.float_output.data(), call.float_output.size()); #endif - for (size_t i = 0; i < int_out.size(); ++i) { - check_int(int_out[i], "library output", i); + for (size_t i = 0; i < call.int_output.size(); ++i) { + check_int(call.int_output[i], "library output", i); } #ifdef GECODE_HAS_FLOAT_VARS - check_floats(float_out, "library output"); + check_floats(call.float_output, "library output"); #endif } @@ -1308,117 +1311,6 @@ BlackBoxExec::~BlackBoxExec(void) { sessions.clear(); } -class BlackBoxState : public SharedHandle::Object { -protected: - class ExecEntry { - public: - std::string program; - std::vector args; - BlackBoxHandle handle; - ExecEntry(const std::string &program0, const std::vector &args0, - const BlackBoxHandle &handle0) - : program(program0), args(args0), handle(handle0) {} - }; - mutable Support::Mutex mutex; - std::vector exec; - std::exception_ptr exception; - std::atomic error_recorded; - -public: - BlackBoxState(void) : error_recorded(false) {} - BlackBoxHandle blackBox(const std::string &mode, - const std::string &instantiation, - const std::vector &args); - void fail(std::exception_ptr e); - bool failed(void) const; - void rethrow(void) const; -}; - -BlackBoxStateHandle -BlackBoxStateHandle::init(SharedHandle &handle) { - BlackBoxStateHandle state(handle); - if (!state) { - state.object(new BlackBoxState); - handle = state; - } - return state; -} - -BlackBoxHandle -BlackBoxStateHandle::blackBox(const std::string &mode, - const std::string &instantiation, - const std::vector &args) const { - return static_cast(object())->blackBox(mode, instantiation, - args); -} - -void -BlackBoxStateHandle::fail(std::exception_ptr e) const { - static_cast(object())->fail(e); -} - -bool -BlackBoxStateHandle::failed(void) const { - return static_cast(*this) && - static_cast(object())->failed(); -} - -void -BlackBoxStateHandle::rethrow(void) const { - if (*this) { - static_cast(object())->rethrow(); - } -} - -BlackBoxHandle -BlackBoxState::blackBox(const std::string &mode, - const std::string &instantiation, - const std::vector &args) { - if (mode == "exec") { - Support::Lock lock(mutex); - for (const ExecEntry &e : exec) { - if ((e.program == instantiation) && (e.args == args)) { - return e.handle; - } - } - BlackBoxHandle handle(new BlackBoxExec(instantiation, args)); - exec.push_back(ExecEntry(instantiation, args, handle)); - return handle; - } - if (mode == "dll") { - return BlackBoxHandle(new BlackBoxLibrary(instantiation, args)); - } - throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); -} - -void -BlackBoxState::fail(std::exception_ptr e) { - Support::Lock lock(mutex); - if (!error_recorded.load(std::memory_order_relaxed)) { - exception = e; - error_recorded.store(true, std::memory_order_release); - } -} - -bool -BlackBoxState::failed(void) const { - return error_recorded.load(std::memory_order_acquire); -} - -void -BlackBoxState::rethrow(void) const { - if (!error_recorded.load(std::memory_order_acquire)) { - return; - } - std::exception_ptr e; - { - Support::Lock lock(mutex); - e = exception; - } - if (e != nullptr) { - std::rethrow_exception(e); - } -} BlackBoxExec::Session &BlackBoxExec::session(void) { Support::Lock lock(mutex); @@ -1434,38 +1326,39 @@ BlackBoxExec::Session &BlackBoxExec::session(void) { return *r; } -void BlackBoxExec::run(const std::vector &int_in, - const std::vector &float_in, - std::vector &int_out, - std::vector &float_out) { +std::string +encode_blackbox_request(const BlackBoxCall& call) { // Construct program input: comma-separated integers, a semicolon, then // comma-separated floats, terminated by a newline (e.g. "5,-7;2.5,1.125\n"). std::ostringstream out; out.imbue(std::locale::classic()); out.precision(std::numeric_limits::max_digits10); - for (size_t i = 0; i < int_in.size(); ++i) { + for (size_t i = 0; i < call.int_input.size(); ++i) { if (i != 0) { out << ","; } - out << int_in[i]; + out << call.int_input[i]; } out << ";"; - for (size_t i = 0; i < float_in.size(); ++i) { + for (size_t i = 0; i < call.float_input.size(); ++i) { if (i != 0) { out << ","; } - out << float_in[i]; + out << call.float_input[i]; } out << "\n"; - std::string out_buf = out.str(); - std::string in_buffer = session().run(out_buf); - if (in_buffer.find('\0') != std::string::npos) { + return out.str(); +} + +void +decode_blackbox_response(const std::string& response, BlackBoxCall& call) { + if (response.find('\0') != std::string::npos) { throw Error("BlackBoxExec", "Blackbox process response contains NUL data."); } // Parse the response in a single left-to-right pass: comma-separated // integers, a semicolon, then comma-separated floats (e.g. "5,-7;2.5,1.125\n"). - const char *p = in_buffer.c_str(); + const char *p = response.c_str(); auto skip_ws = [](const char *&q) { while (*q == ' ' || *q == '\t' || *q == '\r') { ++q; @@ -1500,7 +1393,7 @@ void BlackBoxExec::run(const std::vector &int_in, } return true; }; - for (size_t i = 0; i < int_out.size(); ++i) { + for (size_t i = 0; i < call.int_output.size(); ++i) { skip_ws(p); const char *end = value_end(p); const char *integer = p; @@ -1515,14 +1408,14 @@ void BlackBoxExec::run(const std::vector &int_in, throw Error("BlackBoxExec", "Failed to read output integer " + std::to_string(i) + " from blackbox process output, " + - std::to_string(int_out.size()) + + std::to_string(call.int_output.size()) + " integer values were expected."); } check_int(value, "blackbox process output", i); - int_out[i] = value; + call.int_output[i] = value; p = end; skip_ws(p); - if (i + 1 < int_out.size()) { + if (i + 1 < call.int_output.size()) { if (*p != ',') { throw Error("BlackBoxExec", "Blackbox process response is missing an integer output " @@ -1538,7 +1431,7 @@ void BlackBoxExec::run(const std::vector &int_in, "the integer and floating point outputs."); } ++p; - for (size_t i = 0; i < float_out.size(); ++i) { + for (size_t i = 0; i < call.float_output.size(); ++i) { skip_ws(p); const char *end = value_end(p); std::istringstream in(std::string(p, end)); @@ -1548,16 +1441,16 @@ void BlackBoxExec::run(const std::vector &int_in, throw Error("BlackBoxExec", "Failed to read output float " + std::to_string(i) + " from blackbox process output, " + - std::to_string(float_out.size()) + + std::to_string(call.float_output.size()) + " floating point values were expected."); } #ifdef GECODE_HAS_FLOAT_VARS check_float(v, "blackbox process output", i); #endif - float_out[i] = v; + call.float_output[i] = v; p = end; skip_ws(p); - if (i + 1 < float_out.size()) { + if (i + 1 < call.float_output.size()) { if (*p != ',') { throw Error("BlackBoxExec", "Blackbox process response is missing a floating point " @@ -1573,253 +1466,10 @@ void BlackBoxExec::run(const std::vector &int_in, } } -ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { - if (int_input.assigned() -#ifdef GECODE_HAS_FLOAT_VARS - && float_input.assigned() -#endif - ) { - std::vector int_in(int_input.size()); - std::vector int_out(int_output.size()); - for (size_t i = 0; i < int_in.size(); i++) { - int_in[i] = static_cast(int_input[i].val()); - } - std::vector float_in; - std::vector float_out; -#ifdef GECODE_HAS_FLOAT_VARS - float_in.resize(float_input.size()); - float_out.resize(float_output.size()); - for (int i = 0; i < float_in.size(); i++) { - float_in[i] = float_input[i].val().med(); - } -#endif - - try { - black_box()->run(int_in, float_in, int_out, float_out); - } catch (...) { - black_box_state.fail(std::current_exception()); - return ES_FAILED; - } - - for (size_t i = 0; i < int_out.size(); i++) { - GECODE_ME_CHECK(int_output[i].eq(home, static_cast(int_out[i]))); - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < float_out.size(); i++) { - GECODE_ME_CHECK(float_output[i].eq(home, float_out[i])); - } -#endif - - return home.ES_SUBSUMED(*this); - } - return ES_FIX; -} - -ExecStatus -BlackBoxBounds::evaluate(Home home, ViewArray &ivar, -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray &fvar, -#endif - BlackBoxHandle &black_box, - const BlackBoxStateHandle &black_box_state) { - std::vector int_in(ivar.size() * 2); - std::vector int_out(ivar.size() * 2); - for (int i = 0; i < ivar.size(); i++) { - int_in[i*2] = static_cast(ivar[i].min()); - int_in[i*2+1] = static_cast(ivar[i].max()); - } - std::vector float_in; - std::vector float_out; -#ifdef GECODE_HAS_FLOAT_VARS - float_in.resize(fvar.size() * 2); - float_out.resize(fvar.size() * 2); - for (int i = 0; i < fvar.size(); i++) { - float_in[i*2] = fvar[i].min(); - float_in[i*2+1] = fvar[i].max(); - } -#endif - - try { - black_box()->run(int_in, float_in, int_out, float_out); - } catch (...) { - black_box_state.fail(std::current_exception()); - return ES_FAILED; - } - - for (int i = 0; i < ivar.size(); i++) { - if (me_failed(ivar[i].gq(home, static_cast(int_out[i*2]))) || - me_failed(ivar[i].lq(home, static_cast(int_out[i*2+1])))) { - return ES_FAILED; - } - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < fvar.size(); i++) { - if (me_failed(fvar[i].gq(home, float_out[i*2])) || - me_failed(fvar[i].lq(home, float_out[i*2+1]))) { - return ES_FAILED; - } - } -#endif - - return ES_OK; -} - -ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { - ExecStatus es = evaluate(home, ivar, -#ifdef GECODE_HAS_FLOAT_VARS - fvar, -#endif - black_box, black_box_state); - return (es == ES_OK) ? ES_NOFIX : es; -} - -void blackbox(Home home, SharedHandle &black_box_state, - const IntVarArgs &int_in, const IntVarArgs &int_out, -#ifdef GECODE_HAS_FLOAT_VARS - const FloatVarArgs &float_in, const FloatVarArgs &float_out, -#endif - const std::string &mode, const std::string &instantiation, - const std::vector &args) { - ViewArray int_input(home, int_in); - ViewArray int_output(home, int_out); -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray float_input(home, float_in); - ViewArray float_output(home, float_out); -#endif - - if (home.failed()) - return; - BlackBoxStateHandle state = BlackBoxStateHandle::init(black_box_state); - PostInfo pi(home); - ExecStatus es = BlackBox::post(home, int_input, int_output, -#ifdef GECODE_HAS_FLOAT_VARS - float_input, float_output, -#endif - state, - mode, instantiation, args); - GECODE_ES_FAIL(es); -} - -/// Parse the flat reason and mark, per channel, the variables whose bounds the -/// propagator depends on (the variables that appear as literals in any reason). -/// \a sub_int / \a sub_float are filled with one boolean per variable. Variable -/// indices in the reason are 1-based over the combined variable list, integer -/// variables first, then float variables. -/// -/// The flat reason is a concatenation of one entry per variable, each entry -/// being `[idx, |R_lb|, (var, bnd)..., |R_ub|, (var, bnd)...]`. -static void reason_subscriptions(const std::vector &reason, int n_int, - int n_float, SharedArray &sub_int, - SharedArray &sub_float) { - for (int i = 0; i < n_int; i++) { - sub_int[i] = false; - } - for (int i = 0; i < n_float; i++) { - sub_float[i] = false; - } - - const int n_total = n_int + n_float; - std::vector explained(n_total, false); - size_t pos = 0; - auto require = [&](size_t n, const char *what) { - if (reason.size() - pos < n) { - throw Error("Blackbox", std::string("Malformed blackbox bounds reason: ") + - what + "."); - } - }; - while (pos < reason.size()) { - require(1, "missing explained variable index"); - int idx = reason[pos++]; - if ((idx < 1) || (idx > n_total)) { - throw Error("Blackbox", - "Malformed blackbox bounds reason: explained variable index " - "is out of range."); - } - if (explained[idx - 1]) { - throw Error("Blackbox", - "Malformed blackbox bounds reason: duplicate explained " - "variable index."); - } - explained[idx - 1] = true; - for (int side = 0; side < 2; side++) { // lower- then upper-bound literals - require(1, "missing reason literal count"); - int count = reason[pos++]; - if (count < 0) { - throw Error("Blackbox", - "Malformed blackbox bounds reason: negative literal count."); - } - if (static_cast(count) > (reason.size() - pos) / 2) { - throw Error("Blackbox", - "Malformed blackbox bounds reason: truncated reason " - "literals."); - } - for (int k = 0; k < count; k++) { - int var = reason[pos++]; // 1-based combined variable index - int bnd = reason[pos++]; - if (var >= 1 && var <= n_int) { - sub_int[var - 1] = true; - } else if (var > n_int && var <= n_int + n_float) { - sub_float[var - 1 - n_int] = true; - } else { - throw Error("Blackbox", - "Malformed blackbox bounds reason: dependency variable " - "index is out of range."); - } - if (bnd != 1 && bnd != 2) { // MiniZinc PropBnd: PR_LB, PR_UB - throw Error("Blackbox", - "Malformed blackbox bounds reason: dependency bound " - "code is out of range."); - } - } - } - } - for (int i = 0; i < n_total; i++) { - if (!explained[i]) { - throw Error("Blackbox", - "Malformed blackbox bounds reason: missing explained " - "variable entry."); - } - } -} - -void blackbox_bounds(Home home, SharedHandle &black_box_state, - const IntVarArgs &ivar, -#ifdef GECODE_HAS_FLOAT_VARS - const FloatVarArgs &fvar, -#endif - const std::string &mode, const std::string &instantiation, - const std::vector &args, - const std::vector &reason) { - ViewArray int_var(home, ivar); -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray float_var(home, fvar); - int n_float = fvar.size(); -#else - int n_float = 0; -#endif - - // Determine which variables the propagator depends on, so it is only - // subscribed (and thus scheduled) on the bounds mentioned in the reason. The - // marking is constant and shared between all copies of the propagator. - SharedArray sub_int(ivar.size()); - SharedArray sub_float(n_float); - reason_subscriptions(reason, ivar.size(), n_float, sub_int, sub_float); - - if (home.failed()) - return; - BlackBoxStateHandle state = BlackBoxStateHandle::init(black_box_state); - PostInfo pi(home); - ExecStatus es = BlackBoxBounds::post(home, int_var, -#ifdef GECODE_HAS_FLOAT_VARS - float_var, -#endif - sub_int, -#ifdef GECODE_HAS_FLOAT_VARS - sub_float, -#endif - state, - mode, instantiation, args); - GECODE_ES_FAIL(es); +void +BlackBoxExec::run(BlackBoxCall& call) { + const std::string response = session().run(encode_blackbox_request(call)); + decode_blackbox_response(response, call); } } // namespace FlatZinc diff --git a/gecode/flatzinc/blackbox-backend.hh b/gecode/flatzinc/blackbox-backend.hh new file mode 100644 index 0000000000..b08f97c1ca --- /dev/null +++ b/gecode/flatzinc/blackbox-backend.hh @@ -0,0 +1,135 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Copyright: + * Jip J. Dekker, 2026 + * + * This file is part of Gecode, the generic constraint + * development environment: + * http://www.gecode.org + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef GECODE_FLATZINC_BLACKBOX_BACKEND_HH +#define GECODE_FLATZINC_BLACKBOX_BACKEND_HH + +#include +#include +#include +#include + +#include +#include +#ifdef GECODE_HAS_THREADS +#include +#endif + +#ifdef _WIN32 +#define GECODE_BLACKBOX_CALL __stdcall +#else +#define GECODE_BLACKBOX_CALL +#endif + +namespace Gecode { +namespace FlatZinc { + +/// Inputs and pre-sized output buffers for one backend call. +struct BlackBoxCall { + const std::vector& int_input; + const std::vector& float_input; + std::vector& int_output; + std::vector& float_output; +}; + +/// Backend for a deterministic FlatZinc blackbox function. +class BlackBoxBackend : public SharedHandle::Object { +public: + virtual ~BlackBoxBackend(void) {} + virtual void run(BlackBoxCall& call) = 0; + void run(const std::vector& int_input, + const std::vector& float_input, + std::vector& int_output, + std::vector& float_output) { + BlackBoxCall call = {int_input, float_input, int_output, float_output}; + run(call); + } +}; + +/// Dynamic-library backend owned by one blackbox constraint. +class GECODE_FLATZINC_EXPORT BlackBoxLibrary : public BlackBoxBackend { +public: + using BlackBoxBackend::run; + BlackBoxLibrary(const std::string& name, + const std::vector& args); + ~BlackBoxLibrary(); + void run(BlackBoxCall& call) override; + +protected: + void* library; + void* (GECODE_BLACKBOX_CALL *library_fzn_init)(const char**, size_t); + void* (GECODE_BLACKBOX_CALL *library_fzn_clone)(void*); + void (GECODE_BLACKBOX_CALL *library_fzn_blackbox)( + void*, const int64_t*, size_t, const double*, size_t, int64_t*, size_t, + double*, size_t); + void (GECODE_BLACKBOX_CALL *library_fzn_free)(void*); + void* root_instance; + +#ifdef GECODE_HAS_THREADS + class Instance; + Support::Mutex mutex; + std::vector instances; + Instance* instance(void); +#endif +}; + +/// Persistent-process backend shared by equal executable configurations. +class GECODE_FLATZINC_EXPORT BlackBoxExec : public BlackBoxBackend { +public: + using BlackBoxBackend::run; + BlackBoxExec(const std::string& program, + const std::vector& args); + ~BlackBoxExec(); + void run(BlackBoxCall& call) override; + +protected: + class Session; + std::string program; + std::vector args; + Support::Mutex mutex; + std::vector sessions; + Session& session(void); +}; + +/// Encode one request for the executable backend's line protocol. +GECODE_FLATZINC_EXPORT +std::string encode_blackbox_request(const BlackBoxCall& call); +/// Decode and validate one response from the executable backend. +GECODE_FLATZINC_EXPORT +void decode_blackbox_response(const std::string& response, + BlackBoxCall& call); + +} // namespace FlatZinc +} // namespace Gecode + +#endif // GECODE_FLATZINC_BLACKBOX_BACKEND_HH diff --git a/gecode/flatzinc/blackbox-propagator.cpp b/gecode/flatzinc/blackbox-propagator.cpp new file mode 100644 index 0000000000..28a0fb1e64 --- /dev/null +++ b/gecode/flatzinc/blackbox-propagator.cpp @@ -0,0 +1,718 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Copyright: + * Jip J. Dekker, 2026 + * + * This file is part of Gecode, the generic constraint + * development environment: + * http://www.gecode.org + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include + +#include +#include +#include +#include +#include + +namespace Gecode { +namespace FlatZinc { + +namespace { + +class BlackBoxHandle : public SharedHandle { +public: + explicit BlackBoxHandle(BlackBoxBackend* backend) : SharedHandle() { + object(backend); + } + explicit BlackBoxHandle(const SharedHandle& handle) : SharedHandle(handle) {} + BlackBoxHandle(const BlackBoxHandle& handle) : SharedHandle(handle) {} + BlackBoxHandle& operator=(const BlackBoxHandle& handle) { + return static_cast(SharedHandle::operator=(handle)); + } + BlackBoxBackend* operator()(void) const { + return static_cast(object()); + } +}; + +} + +class BlackBoxContext : public SharedHandle::Object { +protected: + class ExecEntry { + public: + std::string program; + std::vector args; + BlackBoxHandle handle; + ExecEntry(const std::string &program0, const std::vector &args0, + const BlackBoxHandle &handle0) + : program(program0), args(args0), handle(handle0) {} + }; + mutable Support::Mutex mutex; + std::vector exec; + std::exception_ptr exception; + std::atomic error_recorded; + +public: + BlackBoxContext(void) : error_recorded(false) {} + BlackBoxHandle backendForConstraint(const std::string& mode, + const std::string& target, + const std::vector& args); + void fail(std::exception_ptr e); + bool failed(void) const; + void rethrow(void) const; +}; + +void +BlackBoxContextHandle::init(void) { + if (!*this) { + object(new BlackBoxContext); + } +} + +SharedHandle +BlackBoxContextHandle::backendForConstraint( + const std::string& mode, const std::string& target, + const std::vector& args) const { + return static_cast(object()) + ->backendForConstraint(mode, target, args); +} + +void +BlackBoxContextHandle::fail(std::exception_ptr e) const { + static_cast(object())->fail(e); +} + +bool +BlackBoxContextHandle::failed(void) const { + return static_cast(*this) && + static_cast(object())->failed(); +} + +void +BlackBoxContextHandle::rethrow(void) const { + if (*this) { + static_cast(object())->rethrow(); + } +} + +BlackBoxHandle +BlackBoxContext::backendForConstraint(const std::string& mode, + const std::string& target, + const std::vector& args) { + if (mode == "exec") { + Support::Lock lock(mutex); + for (const ExecEntry &e : exec) { + if ((e.program == target) && (e.args == args)) { + return e.handle; + } + } + BlackBoxHandle handle(new BlackBoxExec(target, args)); + exec.push_back(ExecEntry(target, args, handle)); + return handle; + } + if (mode == "dll") { + return BlackBoxHandle(new BlackBoxLibrary(target, args)); + } + throw Error("Blackbox", "Unknown blackbox protocol `" + mode + "'"); +} + +void +BlackBoxContext::fail(std::exception_ptr e) { + Support::Lock lock(mutex); + if (!error_recorded.load(std::memory_order_relaxed)) { + exception = e; + error_recorded.store(true, std::memory_order_release); + } +} + +bool +BlackBoxContext::failed(void) const { + return error_recorded.load(std::memory_order_acquire); +} + +void +BlackBoxContext::rethrow(void) const { + if (!error_recorded.load(std::memory_order_acquire)) { + return; + } + std::exception_ptr e; + { + Support::Lock lock(mutex); + e = exception; + } + if (e != nullptr) { + std::rethrow_exception(e); + } +} + +namespace { + +class BlackBox : public Propagator { +protected: + ViewArray int_input; + ViewArray int_output; +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray float_input; + ViewArray float_output; +#endif + BlackBoxHandle backend; + BlackBoxContextHandle context; + + BlackBox(Space& home, BlackBox& p) + : Propagator(home, p), backend(p.backend), context(p.context) { + int_input.update(home, p.int_input); + int_output.update(home, p.int_output); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.update(home, p.float_input); + float_output.update(home, p.float_output); +#endif + } + +public: + BlackBox(Home home, ViewArray& int_in, + ViewArray& int_out, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray& float_in, + ViewArray& float_out, +#endif + const BlackBoxHandle& backend0, + const BlackBoxContextHandle& context0) + : Propagator(home), int_input(int_in), int_output(int_out), +#ifdef GECODE_HAS_FLOAT_VARS + float_input(float_in), float_output(float_out), +#endif + backend(backend0), context(context0) { + int_input.subscribe(home, *this, Int::PC_INT_VAL); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.subscribe(home, *this, Float::PC_FLOAT_VAL); +#endif + home.notice(*this, AP_DISPOSE); + } + + PropCost cost(const Space&, const ModEventDelta&) const override { + return PropCost::crazy(PropCost::HI, int_input.size() +#ifdef GECODE_HAS_FLOAT_VARS + + float_input.size() +#endif + ); + } + + void reschedule(Space& home) override { + int_input.reschedule(home, *this, Int::PC_INT_VAL); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.reschedule(home, *this, Float::PC_FLOAT_VAL); +#endif + } + + size_t dispose(Space& home) override { + int_input.cancel(home, *this, Int::PC_INT_VAL); +#ifdef GECODE_HAS_FLOAT_VARS + float_input.cancel(home, *this, Float::PC_FLOAT_VAL); +#endif + home.ignore(*this, AP_DISPOSE); + backend.~BlackBoxHandle(); + context.~BlackBoxContextHandle(); + (void) Propagator::dispose(home); + return sizeof(*this); + } + + ExecStatus propagate(Space& home, const ModEventDelta&) override; + + Propagator* copy(Space& home) override { + return new (home) BlackBox(home, *this); + } + + static ExecStatus post(Home home, ViewArray& int_input, + ViewArray& int_output, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray& float_input, + ViewArray& float_output, +#endif + const BlackBoxContextHandle& context, + const std::string& mode, const std::string& target, + const std::vector& args); +}; + +class BlackBoxBounds : public Propagator { +protected: + ViewArray ivar; +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray fvar; +#endif + SharedArray sub_int; +#ifdef GECODE_HAS_FLOAT_VARS + SharedArray sub_float; +#endif + BlackBoxHandle backend; + BlackBoxContextHandle context; + + BlackBoxBounds(Space& home, BlackBoxBounds& p) + : Propagator(home, p), sub_int(p.sub_int), +#ifdef GECODE_HAS_FLOAT_VARS + sub_float(p.sub_float), +#endif + backend(p.backend), context(p.context) { + ivar.update(home, p.ivar); +#ifdef GECODE_HAS_FLOAT_VARS + fvar.update(home, p.fvar); +#endif + } + +public: + BlackBoxBounds(Home home, ViewArray& ivar0, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray& fvar0, +#endif + SharedArray sub_int0, +#ifdef GECODE_HAS_FLOAT_VARS + SharedArray sub_float0, +#endif + const BlackBoxHandle& backend0, + const BlackBoxContextHandle& context0) + : Propagator(home), ivar(ivar0), +#ifdef GECODE_HAS_FLOAT_VARS + fvar(fvar0), +#endif + sub_int(sub_int0), +#ifdef GECODE_HAS_FLOAT_VARS + sub_float(sub_float0), +#endif + backend(backend0), context(context0) { + for (int i = 0; i < ivar.size(); i++) + if (sub_int[i]) + ivar[i].subscribe(home, *this, Int::PC_INT_BND); +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) + if (sub_float[i]) + fvar[i].subscribe(home, *this, Float::PC_FLOAT_BND); +#endif + home.notice(*this, AP_DISPOSE); + home.notice(*this, AP_WEAKLY); + } + + PropCost cost(const Space&, const ModEventDelta&) const override { + return PropCost::crazy(PropCost::HI, ivar.size() +#ifdef GECODE_HAS_FLOAT_VARS + + fvar.size() +#endif + ); + } + + void reschedule(Space& home) override { + for (int i = 0; i < ivar.size(); i++) + if (sub_int[i]) + ivar[i].reschedule(home, *this, Int::PC_INT_BND); +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) + if (sub_float[i]) + fvar[i].reschedule(home, *this, Float::PC_FLOAT_BND); +#endif + } + + size_t dispose(Space& home) override { + for (int i = 0; i < ivar.size(); i++) + if (sub_int[i]) + ivar[i].cancel(home, *this, Int::PC_INT_BND); +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) + if (sub_float[i]) + fvar[i].cancel(home, *this, Float::PC_FLOAT_BND); +#endif + home.ignore(*this, AP_DISPOSE); + home.ignore(*this, AP_WEAKLY); + backend.~BlackBoxHandle(); + context.~BlackBoxContextHandle(); + sub_int.~SharedArray(); +#ifdef GECODE_HAS_FLOAT_VARS + sub_float.~SharedArray(); +#endif + (void) Propagator::dispose(home); + return sizeof(*this); + } + + ExecStatus propagate(Space& home, const ModEventDelta&) override; + Propagator* copy(Space& home) override { + return new (home) BlackBoxBounds(home, *this); + } + + static ExecStatus evaluate(Home home, ViewArray& ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray& fvar, +#endif + BlackBoxHandle& backend, + const BlackBoxContextHandle& context); + static ExecStatus post(Home home, ViewArray& ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray& fvar, +#endif + SharedArray sub_int, +#ifdef GECODE_HAS_FLOAT_VARS + SharedArray sub_float, +#endif + const BlackBoxContextHandle& context, + const std::string& mode, const std::string& target, + const std::vector& args); +}; + +} // namespace + +ExecStatus +BlackBox::post(Home home, ViewArray& int_input, + ViewArray& int_output, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray& float_input, + ViewArray& float_output, +#endif + const BlackBoxContextHandle& context, + const std::string& mode, const std::string& target, + const std::vector& args) { + BlackBoxHandle backend(context.backendForConstraint(mode, target, args)); + if ((int_input.size() == 0) +#ifdef GECODE_HAS_FLOAT_VARS + && (float_input.size() == 0) +#endif + ) { + std::vector int_in; + std::vector int_out(int_output.size()); + std::vector float_in; + std::vector float_out; +#ifdef GECODE_HAS_FLOAT_VARS + float_out.resize(float_output.size()); +#endif + BlackBoxCall call = {int_in, float_in, int_out, float_out}; + backend()->run(call); + for (int i = 0; i < int_output.size(); i++) + if (me_failed(int_output[i].eq(home, static_cast(int_out[i])))) + return ES_FAILED; +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < float_output.size(); i++) + if (me_failed(float_output[i].eq(home, float_out[i]))) + return ES_FAILED; +#endif + return ES_OK; + } + + new (home) BlackBox(home, int_input, int_output, +#ifdef GECODE_HAS_FLOAT_VARS + float_input, float_output, +#endif + backend, context); + return ES_OK; +} + +ExecStatus +BlackBoxBounds::post(Home home, ViewArray& ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray& fvar, +#endif + SharedArray sub_int, +#ifdef GECODE_HAS_FLOAT_VARS + SharedArray sub_float, +#endif + const BlackBoxContextHandle& context, + const std::string& mode, const std::string& target, + const std::vector& args) { + BlackBoxHandle backend(context.backendForConstraint(mode, target, args)); + bool has_subscription = false; + for (int i = 0; i < ivar.size(); i++) + has_subscription = has_subscription || sub_int[i]; +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) + has_subscription = has_subscription || sub_float[i]; +#endif + if (!has_subscription) + return evaluate(home, ivar, +#ifdef GECODE_HAS_FLOAT_VARS + fvar, +#endif + backend, context); + + new (home) BlackBoxBounds(home, ivar, +#ifdef GECODE_HAS_FLOAT_VARS + fvar, +#endif + sub_int, +#ifdef GECODE_HAS_FLOAT_VARS + sub_float, +#endif + backend, context); + return ES_OK; +} + +ExecStatus BlackBox::propagate(Space &home, const ModEventDelta &) { + if (int_input.assigned() +#ifdef GECODE_HAS_FLOAT_VARS + && float_input.assigned() +#endif + ) { + std::vector int_in(int_input.size()); + std::vector int_out(int_output.size()); + for (size_t i = 0; i < int_in.size(); i++) { + int_in[i] = static_cast(int_input[i].val()); + } + std::vector float_in; + std::vector float_out; +#ifdef GECODE_HAS_FLOAT_VARS + float_in.resize(float_input.size()); + float_out.resize(float_output.size()); + for (size_t i = 0; i < float_in.size(); i++) { + float_in[i] = float_input[i].val().med(); + } +#endif + + try { + BlackBoxCall call = {int_in, float_in, int_out, float_out}; + backend()->run(call); + } catch (...) { + context.fail(std::current_exception()); + return ES_FAILED; + } + + for (size_t i = 0; i < int_out.size(); i++) { + GECODE_ME_CHECK(int_output[i].eq(home, static_cast(int_out[i]))); + } +#ifdef GECODE_HAS_FLOAT_VARS + for (size_t i = 0; i < float_out.size(); i++) { + GECODE_ME_CHECK(float_output[i].eq(home, float_out[i])); + } +#endif + + return home.ES_SUBSUMED(*this); + } + return ES_FIX; +} + +ExecStatus +BlackBoxBounds::evaluate(Home home, ViewArray &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray &fvar, +#endif + BlackBoxHandle& backend, + const BlackBoxContextHandle& context) { + std::vector int_in(ivar.size() * 2); + std::vector int_out(ivar.size() * 2); + for (int i = 0; i < ivar.size(); i++) { + int_in[i*2] = static_cast(ivar[i].min()); + int_in[i*2+1] = static_cast(ivar[i].max()); + } + std::vector float_in; + std::vector float_out; +#ifdef GECODE_HAS_FLOAT_VARS + float_in.resize(fvar.size() * 2); + float_out.resize(fvar.size() * 2); + for (int i = 0; i < fvar.size(); i++) { + float_in[i*2] = fvar[i].min(); + float_in[i*2+1] = fvar[i].max(); + } +#endif + + try { + BlackBoxCall call = {int_in, float_in, int_out, float_out}; + backend()->run(call); + } catch (...) { + context.fail(std::current_exception()); + return ES_FAILED; + } + + for (int i = 0; i < ivar.size(); i++) { + if (me_failed(ivar[i].gq(home, static_cast(int_out[i*2]))) || + me_failed(ivar[i].lq(home, static_cast(int_out[i*2+1])))) { + return ES_FAILED; + } + } +#ifdef GECODE_HAS_FLOAT_VARS + for (int i = 0; i < fvar.size(); i++) { + if (me_failed(fvar[i].gq(home, float_out[i*2])) || + me_failed(fvar[i].lq(home, float_out[i*2+1]))) { + return ES_FAILED; + } + } +#endif + + return ES_OK; +} + +ExecStatus BlackBoxBounds::propagate(Space &home, const ModEventDelta &) { + ExecStatus es = evaluate(home, ivar, +#ifdef GECODE_HAS_FLOAT_VARS + fvar, +#endif + backend, context); + return (es == ES_OK) ? ES_NOFIX : es; +} + +void blackbox(Home home, BlackBoxContextHandle& context, + const IntVarArgs &int_in, const IntVarArgs &int_out, +#ifdef GECODE_HAS_FLOAT_VARS + const FloatVarArgs &float_in, const FloatVarArgs &float_out, +#endif + const std::string &mode, const std::string &target, + const std::vector &args) { + ViewArray int_input(home, int_in); + ViewArray int_output(home, int_out); +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray float_input(home, float_in); + ViewArray float_output(home, float_out); +#endif + + if (home.failed()) + return; + context.init(); + PostInfo pi(home); + ExecStatus es = BlackBox::post(home, int_input, int_output, +#ifdef GECODE_HAS_FLOAT_VARS + float_input, float_output, +#endif + context, mode, target, args); + GECODE_ES_FAIL(es); +} + +/// Parse the flat reason and mark, per channel, the variables whose bounds the +/// propagator depends on (the variables that appear as literals in any reason). +/// \a sub_int / \a sub_float are filled with one boolean per variable. Variable +/// indices in the reason are 1-based over the combined variable list, integer +/// variables first, then float variables. +/// +/// The flat reason is a concatenation of one entry per variable, each entry +/// being `[idx, |R_lb|, (var, bnd)..., |R_ub|, (var, bnd)...]`. +static void reason_subscriptions(const std::vector &reason, int n_int, + int n_float, SharedArray &sub_int, + SharedArray &sub_float) { + for (int i = 0; i < n_int; i++) { + sub_int[i] = false; + } + for (int i = 0; i < n_float; i++) { + sub_float[i] = false; + } + + const int n_total = n_int + n_float; + std::vector explained(n_total, false); + size_t pos = 0; + auto require = [&](size_t n, const char *what) { + if (reason.size() - pos < n) { + throw Error("Blackbox", std::string("Malformed blackbox bounds reason: ") + + what + "."); + } + }; + while (pos < reason.size()) { + require(1, "missing explained variable index"); + int idx = reason[pos++]; + if ((idx < 1) || (idx > n_total)) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: explained variable index " + "is out of range."); + } + if (explained[idx - 1]) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: duplicate explained " + "variable index."); + } + explained[idx - 1] = true; + for (int side = 0; side < 2; side++) { // lower- then upper-bound literals + require(1, "missing reason literal count"); + int count = reason[pos++]; + if (count < 0) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: negative literal count."); + } + if (static_cast(count) > (reason.size() - pos) / 2) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: truncated reason " + "literals."); + } + for (int k = 0; k < count; k++) { + int var = reason[pos++]; // 1-based combined variable index + int bnd = reason[pos++]; + if (var >= 1 && var <= n_int) { + sub_int[var - 1] = true; + } else if (var > n_int && var <= n_int + n_float) { + sub_float[var - 1 - n_int] = true; + } else { + throw Error("Blackbox", + "Malformed blackbox bounds reason: dependency variable " + "index is out of range."); + } + if (bnd != 1 && bnd != 2) { // MiniZinc PropBnd: PR_LB, PR_UB + throw Error("Blackbox", + "Malformed blackbox bounds reason: dependency bound " + "code is out of range."); + } + } + } + } + for (int i = 0; i < n_total; i++) { + if (!explained[i]) { + throw Error("Blackbox", + "Malformed blackbox bounds reason: missing explained " + "variable entry."); + } + } +} + +void blackbox_bounds(Home home, BlackBoxContextHandle& context, + const IntVarArgs &ivar, +#ifdef GECODE_HAS_FLOAT_VARS + const FloatVarArgs &fvar, +#endif + const std::string &mode, const std::string &target, + const std::vector &args, + const std::vector &reason) { + ViewArray int_var(home, ivar); +#ifdef GECODE_HAS_FLOAT_VARS + ViewArray float_var(home, fvar); + int n_float = fvar.size(); +#else + int n_float = 0; +#endif + + // Determine which variables the propagator depends on, so it is only + // subscribed (and thus scheduled) on the bounds mentioned in the reason. The + // marking is constant and shared between all copies of the propagator. + SharedArray sub_int(ivar.size()); + SharedArray sub_float(n_float); + reason_subscriptions(reason, ivar.size(), n_float, sub_int, sub_float); + + if (home.failed()) + return; + context.init(); + PostInfo pi(home); + ExecStatus es = BlackBoxBounds::post(home, int_var, +#ifdef GECODE_HAS_FLOAT_VARS + float_var, +#endif + sub_int, +#ifdef GECODE_HAS_FLOAT_VARS + sub_float, +#endif + context, mode, target, args); + GECODE_ES_FAIL(es); +} + +} // namespace FlatZinc +} // namespace Gecode diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index 1f40ebf7c5..c8cbbb83fa 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -34,8 +34,6 @@ #ifndef GECODE_FLATZINC_BLACKBOX_HH #define GECODE_FLATZINC_BLACKBOX_HH -#include -#include #include #include #include @@ -45,491 +43,59 @@ #ifdef GECODE_HAS_FLOAT_VARS #include #endif -#ifdef GECODE_HAS_THREADS -#include -#endif - -#ifdef _WIN32 -#define GECODE_BLACKBOX_CALL __stdcall -#else -#define GECODE_BLACKBOX_CALL -#endif namespace Gecode { namespace FlatZinc { -/// Abstract class implemented by different methods to run blackbox functions. -/// -/// A blackbox function must be deterministic in the FlatZinc sense: the same -/// integer and float inputs must always produce the same integer and float -/// outputs. Implementations may keep internal caches or other private state, -/// provided that this state does not make the observable result depend on call -/// order. -class BlackBoxFn : public SharedHandle::Object { -public: - virtual ~BlackBoxFn(void) {} - virtual void run(const std::vector &int_in, - const std::vector &float_in, - std::vector &int_out, - std::vector &float_out) = 0; -}; - -/// Access to FlatZincSpace state used only while posting blackbox constraints -class BlackBoxAccess { -public: - static SharedHandle& state(FlatZincSpace& s); -}; - -/// Implementation of a black box function that dynamically loads a library and -/// run a contained function. -/// -/// A library backend belongs to one blackbox constraint. If the library exports -/// fzn_init, it creates a root instance for that constraint. With threads, the -/// root is a prototype: each calling thread receives and reuses its own clone, -/// and the same clone is never used by concurrent calls. Without threads, -/// fzn_blackbox receives the root instance. A library without fzn_init is -/// stateless: fzn_blackbox receives a null instance and can be called -/// concurrently. -class GECODE_FLATZINC_EXPORT BlackBoxLibrary : public BlackBoxFn { -public: - BlackBoxLibrary(const std::string &name, - const std::vector &args); - ~BlackBoxLibrary(); - void run(const std::vector &int_in, - const std::vector &float_in, - std::vector &int_out, - std::vector &float_out) override; - -protected: - void *library; - void *(GECODE_BLACKBOX_CALL *library_fzn_init)(const char **, size_t); - void *(GECODE_BLACKBOX_CALL *library_fzn_clone)(void *); - void (GECODE_BLACKBOX_CALL *library_fzn_blackbox)( - void *, const int64_t *, size_t, const double *, size_t, int64_t *, - size_t, double *, size_t); - void (GECODE_BLACKBOX_CALL *library_fzn_free)(void *); - void *root_instance; - -#ifdef GECODE_HAS_THREADS - class Instance; - /// Mutex protecting the worker-to-instance table. - Support::Mutex mutex; - /// Cloned instances, one for each calling worker. - std::vector instances; - - Instance *instance(void); -#endif -}; +class BlackBoxContext; -/// Implementation of a blackbox function that starts a separate process to -/// repeatedly run a blackbox function, communicating over standard I/O. -/// -/// Parallel search workers do not share a process stream: each calling thread -/// gets its own persistent process session, created lazily and reused by that -/// thread until the shared blackbox object is destroyed. -class GECODE_FLATZINC_EXPORT BlackBoxExec : public BlackBoxFn { +/// Model-local context shared by blackbox propagators and search support. +class BlackBoxContextHandle : public SharedHandle { public: - BlackBoxExec(const std::string &program, const std::vector &args); - ~BlackBoxExec(); - void run(const std::vector &int_in, - const std::vector &float_in, - std::vector &int_out, - std::vector &float_out) override; - -protected: - class Session; - - /// The executable to run for each worker-thread session. - std::string program; - /// Arguments passed to each executable session. - std::vector args; - /// Mutex protecting the session table. - Support::Mutex mutex; - /// One persistent process session for each calling thread. - std::vector sessions; - - Session &session(void); -}; - -class BlackBoxHandle : public SharedHandle { -public: - BlackBoxHandle(BlackBoxFn *fn) : SharedHandle() { object(fn); } - BlackBoxHandle(const BlackBoxHandle &handle) : SharedHandle(handle) {} - BlackBoxHandle &operator=(const BlackBoxHandle &handle) { - return static_cast(SharedHandle::operator=(handle)); + BlackBoxContextHandle(void) : SharedHandle() {} + BlackBoxContextHandle(const BlackBoxContextHandle& handle) + : SharedHandle(handle) {} + BlackBoxContextHandle& operator=(const BlackBoxContextHandle& handle) { + return static_cast( + SharedHandle::operator=(handle)); } - BlackBoxFn *operator()() { return static_cast(object()); }; -}; -/// Typed handle retained by blackbox propagators and search support. -class BlackBoxStateHandle : public SharedHandle { -public: - BlackBoxStateHandle(void) : SharedHandle() {} - BlackBoxStateHandle(const SharedHandle &handle) : SharedHandle(handle) {} - BlackBoxStateHandle(const BlackBoxStateHandle &handle) - : SharedHandle(handle) {} - BlackBoxStateHandle &operator=(const BlackBoxStateHandle &handle) { - return static_cast(SharedHandle::operator=(handle)); - } - - /// Initialize the model-local state held by \a handle, if necessary - static BlackBoxStateHandle init(SharedHandle &handle); - /// Return a cached blackbox backend, creating it if necessary - BlackBoxHandle blackBox(const std::string &mode, - const std::string &instantiation, - const std::vector &args) const; - /// Record the first exception raised while propagating a blackbox + /// Initialize this context if it is empty. + void init(void); + /// Return the backend selected for one constraint as an opaque handle. + SharedHandle backendForConstraint(const std::string& mode, + const std::string& target, + const std::vector& args) const; + /// Record the first exception raised during blackbox propagation. void fail(std::exception_ptr e) const; - /// Whether a blackbox propagator has failed with an exception + /// Whether blackbox propagation raised an exception. bool failed(void) const; - /// Rethrow the first exception raised by a propagating blackbox + /// Rethrow the first recorded propagation exception. void rethrow(void) const; }; -class BlackBox : public Propagator { -protected: - /// Integer variables considered as the integer input to the blackbox function - ViewArray int_input; - /// Integer variables set to the integer output of the blackbox function - ViewArray int_output; - -#ifdef GECODE_HAS_FLOAT_VARS - /// Floating-point variables considered as the floating-point input to the - /// blackbox function - ViewArray float_input; - /// Floating-point variables set to the floating-point output of the blackbox - /// function - ViewArray float_output; -#endif - - /// Handle to the implementation of the blackbox function - /// - /// The handle ensures that the function implementation can be shared between - /// copies of the propagator. - BlackBoxHandle black_box; - /// State shared by all blackbox propagators in the model - BlackBoxStateHandle black_box_state; - - /// Constructor for cloning \a p - BlackBox(Space &home, BlackBox &p) - : Propagator(home, p), black_box(p.black_box), - black_box_state(p.black_box_state) { - int_input.update(home, p.int_input); - int_output.update(home, p.int_output); -#ifdef GECODE_HAS_FLOAT_VARS - float_input.update(home, p.float_input); - float_output.update(home, p.float_output); -#endif - } - -public: - /// Constructor for creation - BlackBox(Home home, ViewArray &int_in, - ViewArray &int_out, -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray &float_in, - ViewArray &float_out, -#endif - const BlackBoxHandle &black_box0, - const BlackBoxStateHandle &black_box_state0) - : Propagator(home), int_input(int_in), int_output(int_out), -#ifdef GECODE_HAS_FLOAT_VARS - float_input(float_in), float_output(float_out), -#endif - black_box(black_box0), black_box_state(black_box_state0) { - int_input.subscribe(home, *this, Int::PC_INT_VAL); -#ifdef GECODE_HAS_FLOAT_VARS - float_input.subscribe(home, *this, Float::PC_FLOAT_VAL); -#endif - home.notice(*this, AP_DISPOSE); - } - /// Cost function (defined as exponential) - PropCost cost(const Space &, const ModEventDelta &) const override { - return PropCost::crazy(PropCost::HI, int_input.size() -#ifdef GECODE_HAS_FLOAT_VARS - + float_input.size() -#endif - ); - }; - /// Schedule function - void reschedule(Space &home) override { - int_input.reschedule(home, *this, Int::PC_INT_VAL); -#ifdef GECODE_HAS_FLOAT_VARS - float_input.reschedule(home, *this, Float::PC_FLOAT_VAL); -#endif - } - /// Delete propagator and return its size - size_t dispose(Space &home) override { - int_input.cancel(home, *this, Int::PC_INT_VAL); -#ifdef GECODE_HAS_FLOAT_VARS - float_input.cancel(home, *this, Float::PC_FLOAT_VAL); -#endif - home.ignore(*this, AP_DISPOSE); - black_box.~BlackBoxHandle(); - black_box_state.~BlackBoxStateHandle(); - (void)Propagator::dispose(home); - return sizeof(*this); - }; - - ExecStatus propagate(Space &home, const ModEventDelta &) override; - - Propagator *copy(Space &home) override { - return new (home) BlackBox(home, *this); - } - - static ExecStatus post(Home home, ViewArray &int_input, - ViewArray &int_output, -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray &float_input, - ViewArray &float_output, -#endif - const BlackBoxStateHandle &black_box_state, - const std::string &mode, - const std::string &instantiation, - const std::vector &args) { - BlackBoxHandle black_box_handle = - black_box_state.blackBox(mode, instantiation, args); - if ((int_input.size() == 0) -#ifdef GECODE_HAS_FLOAT_VARS - && (float_input.size() == 0) -#endif - ) { - std::vector int_in; - std::vector int_out(int_output.size()); - std::vector float_in; - std::vector float_out; -#ifdef GECODE_HAS_FLOAT_VARS - float_out.resize(float_output.size()); -#endif - black_box_handle()->run(int_in, float_in, int_out, float_out); - for (int i = 0; i < int_output.size(); i++) { - if (me_failed(int_output[i].eq( - home, static_cast(int_out[i])))) { - return ES_FAILED; - } - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < float_output.size(); i++) { - if (me_failed(float_output[i].eq(home, float_out[i]))) { - return ES_FAILED; - } - } -#endif - return ES_OK; - } - - new (home) BlackBox(home, int_input, int_output, -#ifdef GECODE_HAS_FLOAT_VARS - float_input, float_output, -#endif - black_box_handle, black_box_state); - return ES_OK; - } -}; - -class BlackBoxBounds : public Propagator { -protected: - /// Integer variables whose bounds are input and computed by the blackbox - /// function, in order. - ViewArray ivar; - -#ifdef GECODE_HAS_FLOAT_VARS - /// Floating-point variables whose bounds are input and computed by the - /// blackbox function, in order. - ViewArray fvar; -#endif - - /// For each variable in \a ivar, whether the propagator depends on its bounds - /// (derived from the reason). Only marked variables are subscribed, so the - /// propagator is scheduled precisely when one of the relevant bounds changes. - /// - /// The marking is constant during search and is shared between all copies of - /// the propagator. - SharedArray sub_int; -#ifdef GECODE_HAS_FLOAT_VARS - /// For each variable in \a fvar, whether the propagator depends on its bounds. - SharedArray sub_float; -#endif - - /// Handle to the implementation of the blackbox function - /// - /// The handle ensures that the function implementation can be shared between - /// copies of the propagator. - BlackBoxHandle black_box; - /// State shared by all blackbox propagators in the model - BlackBoxStateHandle black_box_state; - - /// Constructor for cloning \a p - BlackBoxBounds(Space &home, BlackBoxBounds &p) - : Propagator(home, p), sub_int(p.sub_int), -#ifdef GECODE_HAS_FLOAT_VARS - sub_float(p.sub_float), -#endif - black_box(p.black_box), black_box_state(p.black_box_state) { - ivar.update(home, p.ivar); -#ifdef GECODE_HAS_FLOAT_VARS - fvar.update(home, p.fvar); -#endif - } - +/// Access to the model-local blackbox context while posting and searching. +class BlackBoxAccess { public: - /// Constructor for creation - BlackBoxBounds(Home home, ViewArray &ivar, -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray &fvar, -#endif - SharedArray sub_int0, -#ifdef GECODE_HAS_FLOAT_VARS - SharedArray sub_float0, -#endif - const BlackBoxHandle &black_box0, - const BlackBoxStateHandle &black_box_state0) - : Propagator(home), ivar(ivar), -#ifdef GECODE_HAS_FLOAT_VARS - fvar(fvar), -#endif - sub_int(sub_int0), -#ifdef GECODE_HAS_FLOAT_VARS - sub_float(sub_float0), -#endif - black_box(black_box0), black_box_state(black_box_state0) { - for (int i = 0; i < ivar.size(); i++) { - if (sub_int[i]) { - ivar[i].subscribe(home, *this, Int::PC_INT_BND); - } - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < fvar.size(); i++) { - if (sub_float[i]) { - fvar[i].subscribe(home, *this, Float::PC_FLOAT_BND); - } - } -#endif - home.notice(*this, AP_DISPOSE); - home.notice(*this, AP_WEAKLY); - } - /// Cost function (defined as exponential) - PropCost cost(const Space &, const ModEventDelta &) const override { - return PropCost::crazy(PropCost::HI, ivar.size() -#ifdef GECODE_HAS_FLOAT_VARS - + fvar.size() -#endif - ); - }; - /// Schedule function - void reschedule(Space &home) override { - for (int i = 0; i < ivar.size(); i++) { - if (sub_int[i]) { - ivar[i].reschedule(home, *this, Int::PC_INT_BND); - } - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < fvar.size(); i++) { - if (sub_float[i]) { - fvar[i].reschedule(home, *this, Float::PC_FLOAT_BND); - } - } -#endif - } - /// Delete propagator and return its size - size_t dispose(Space &home) override { - for (int i = 0; i < ivar.size(); i++) { - if (sub_int[i]) { - ivar[i].cancel(home, *this, Int::PC_INT_BND); - } - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < fvar.size(); i++) { - if (sub_float[i]) { - fvar[i].cancel(home, *this, Float::PC_FLOAT_BND); - } - } -#endif - home.ignore(*this, AP_DISPOSE); - home.ignore(*this, AP_WEAKLY); - black_box.~BlackBoxHandle(); - black_box_state.~BlackBoxStateHandle(); - sub_int.~SharedArray(); -#ifdef GECODE_HAS_FLOAT_VARS - sub_float.~SharedArray(); -#endif - (void)Propagator::dispose(home); - return sizeof(*this); - }; - - ExecStatus propagate(Space &home, const ModEventDelta &) override; - - static ExecStatus evaluate(Home home, ViewArray &ivar, -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray &fvar, -#endif - BlackBoxHandle &black_box, - const BlackBoxStateHandle &black_box_state); - - Propagator *copy(Space &home) override { - return new (home) BlackBoxBounds(home, *this); - } - - static ExecStatus post(Home home, ViewArray &ivar, -#ifdef GECODE_HAS_FLOAT_VARS - ViewArray &fvar, -#endif - SharedArray sub_int, -#ifdef GECODE_HAS_FLOAT_VARS - SharedArray sub_float, -#endif - const BlackBoxStateHandle &black_box_state, - const std::string &mode, - const std::string &instantiation, - const std::vector &args) { - BlackBoxHandle black_box_handle = - black_box_state.blackBox(mode, instantiation, args); - bool has_subscription = false; - for (int i = 0; i < ivar.size(); i++) { - has_subscription = has_subscription || sub_int[i]; - } -#ifdef GECODE_HAS_FLOAT_VARS - for (int i = 0; i < fvar.size(); i++) { - has_subscription = has_subscription || sub_float[i]; - } -#endif - if (!has_subscription) { - return evaluate(home, ivar, -#ifdef GECODE_HAS_FLOAT_VARS - fvar, -#endif - black_box_handle, black_box_state); - } - - new (home) BlackBoxBounds(home, ivar, -#ifdef GECODE_HAS_FLOAT_VARS - fvar, -#endif - sub_int, -#ifdef GECODE_HAS_FLOAT_VARS - sub_float, -#endif - black_box_handle, black_box_state); - return ES_OK; - } + static BlackBoxContextHandle& context(FlatZincSpace& s); }; -void blackbox(Home home, SharedHandle &black_box_state, - const IntVarArgs &int_in, const IntVarArgs &int_out, +void blackbox(Home home, BlackBoxContextHandle& context, + const IntVarArgs& int_in, const IntVarArgs& int_out, #ifdef GECODE_HAS_FLOAT_VARS - const FloatVarArgs &float_in, const FloatVarArgs &float_out, + const FloatVarArgs& float_in, const FloatVarArgs& float_out, #endif - const std::string &mode, const std::string &instantiation, - const std::vector &args); + const std::string& mode, const std::string& target, + const std::vector& args); -void blackbox_bounds(Home home, SharedHandle &black_box_state, - const IntVarArgs &ivar, +void blackbox_bounds(Home home, BlackBoxContextHandle& context, + const IntVarArgs& ivar, #ifdef GECODE_HAS_FLOAT_VARS - const FloatVarArgs &fvar, + const FloatVarArgs& fvar, #endif - const std::string &mode, const std::string &instantiation, - const std::vector &args, - const std::vector &reason); + const std::string& mode, const std::string& target, + const std::vector& args, + const std::vector& reason); } // namespace FlatZinc } // namespace Gecode diff --git a/gecode/flatzinc/flatzinc.cpp b/gecode/flatzinc/flatzinc.cpp index 43172b1152..e66de9c755 100644 --- a/gecode/flatzinc/flatzinc.cpp +++ b/gecode/flatzinc/flatzinc.cpp @@ -49,6 +49,7 @@ #include #include #include +#include #include @@ -771,7 +772,7 @@ namespace Gecode { namespace FlatZinc { DFASet dfaSet; /// Opaque state shared by blackbox propagators in this model - SharedHandle blackBoxState; + BlackBoxContextHandle blackBoxContext; /// Initialize FlatZincSpaceInitData(void) {} @@ -868,10 +869,10 @@ namespace Gecode { namespace FlatZinc { branchInfo.init(); } - SharedHandle& - BlackBoxAccess::state(FlatZincSpace& s) { + BlackBoxContextHandle& + BlackBoxAccess::context(FlatZincSpace& s) { assert(s._initData != nullptr); - return s._initData->blackBoxState; + return s._initData->blackBoxContext; } void @@ -1751,18 +1752,29 @@ namespace Gecode { namespace FlatZinc { class FlatZincStop : public Search::Stop { protected: - Search::Stop* stop_object; - BlackBoxStateHandle black_box_state; + std::unique_ptr stop_object; + BlackBoxContextHandle black_box_context; public: FlatZincStop(Search::Stop* stop_object0, - const BlackBoxStateHandle& black_box_state0) - : stop_object(stop_object0), black_box_state(black_box_state0) {} + const BlackBoxContextHandle& black_box_context0) + : stop_object(stop_object0), black_box_context(black_box_context0) {} bool stop(const Search::Statistics& s, const Search::Options& o) override { - return black_box_state.failed() || - ((stop_object != nullptr) && stop_object->stop(s,o)); + return black_box_context.failed() || + ((stop_object.get() != nullptr) && stop_object->stop(s,o)); } - ~FlatZincStop(void) { - delete stop_object; + }; + + class InterruptHandlerGuard { + protected: + bool installed; + public: + explicit InterruptHandlerGuard(bool install) : installed(install) { + if (installed) + Driver::CombinedStop::installCtrlHandler(true); + } + ~InterruptHandlerGuard(void) { + if (installed) + Driver::CombinedStop::installCtrlHandler(false); } }; @@ -1861,13 +1873,13 @@ namespace Gecode { namespace FlatZinc { const FlatZincOptions& opt, Support::Timer& t_total) { #ifdef GECODE_HAS_GIST if (opt.mode() == SM_GIST) { - BlackBoxStateHandle black_box_state(BlackBoxAccess::state(*this)); + BlackBoxContextHandle& black_box_context = BlackBoxAccess::context(*this); (void) status(); - black_box_state.rethrow(); + black_box_context.rethrow(); FZPrintingInspector pi(p); FZPrintingComparator pc(p); (void) GistEngine >::explore(this,opt,&pi,&pc); - black_box_state.rethrow(); + black_box_context.rethrow(); return; } #endif @@ -1878,14 +1890,17 @@ namespace Gecode { namespace FlatZinc { if (status(sstat) != SS_FAILED) { n_p = PropagatorGroup::all.size(*this); } - BlackBoxStateHandle black_box_state(BlackBoxAccess::state(*this)); - black_box_state.rethrow(); + BlackBoxContextHandle& black_box_context = BlackBoxAccess::context(*this); + black_box_context.rethrow(); Search::Options o; - o.stop = Driver::CombinedStop::create(opt.node(), opt.fail(), opt.time(), - opt.restart_limit(), true); - if (black_box_state) { - o.stop = new FlatZincStop(o.stop, black_box_state); - } + std::unique_ptr stop( + Driver::CombinedStop::create(opt.node(), opt.fail(), opt.time(), + opt.restart_limit(), true)); + if (black_box_context) { + stop.reset(new FlatZincStop(stop.release(), black_box_context)); + } + o.stop = stop.get(); + std::unique_ptr tracer; o.c_d = opt.c_d(); o.a_d = opt.a_d(); @@ -1894,9 +1909,10 @@ namespace Gecode { namespace FlatZinc { FlatZincGetInfo* getInfo = nullptr; if (opt.profiler_info()) getInfo = new FlatZincGetInfo(p); - o.tracer = new CPProfilerSearchTracer(opt.profiler_id(), - opt.name(), opt.profiler_port(), - getInfo); + tracer.reset(new CPProfilerSearchTracer(opt.profiler_id(), + opt.name(), opt.profiler_port(), + getInfo)); + o.tracer = tracer.get(); } #endif @@ -1907,8 +1923,6 @@ namespace Gecode { namespace FlatZinc { o.threads = opt.threads(); o.nogoods_limit = opt.nogoods() ? opt.nogoods_limit() : 0; o.cutoff = new Search::CutoffAppend(new Search::CutoffConstant(0), 1, Driver::createCutoff(opt)); - if (opt.interrupt()) - Driver::CombinedStop::installCtrlHandler(true); int noOfSolutions = opt.solutions(); if (noOfSolutions == -1) { noOfSolutions = (_method == SAT) ? 1 : 0; @@ -1918,46 +1932,33 @@ namespace Gecode { namespace FlatZinc { bool solution_limit_reached = false; bool engine_stopped = false; Gecode::Search::Statistics stat; - FlatZincSpace* sol = nullptr; - try { - { - Meta se(this,o); - while (FlatZincSpace* next_sol = se.next()) { - if (black_box_state.failed()) { - delete next_sol; - break; - } - delete sol; - sol = next_sol; - if (printAll) { - sol->print(out, p); - out << "----------" << std::endl; - } - if (--findSol == 0) { - solution_limit_reached = true; - break; - } + std::unique_ptr sol; + { + InterruptHandlerGuard interrupt_handler(opt.interrupt()); + Meta se(this,o); + while (FlatZincSpace* next = se.next()) { + std::unique_ptr next_sol(next); + if (black_box_context.failed()) { + break; + } + sol = std::move(next_sol); + if (printAll) { + sol->print(out, p); + out << "----------" << std::endl; } - engine_stopped = se.stopped(); - if (opt.mode() == SM_STAT) { - stat = se.statistics(); + if (--findSol == 0) { + solution_limit_reached = true; + break; } } - } catch (...) { - delete sol; - if (opt.interrupt()) - Driver::CombinedStop::installCtrlHandler(false); - delete o.stop; - delete o.tracer; - throw; - } - if (opt.interrupt()) - Driver::CombinedStop::installCtrlHandler(false); - delete o.stop; - delete o.tracer; - if (black_box_state.failed()) { - delete sol; - black_box_state.rethrow(); + engine_stopped = se.stopped(); + if (opt.mode() == SM_STAT) { + stat = se.statistics(); + } + } + if (black_box_context.failed()) { + sol.reset(); + black_box_context.rethrow(); } if (sol && !printAll) { sol->print(out, p); @@ -1974,7 +1975,6 @@ namespace Gecode { namespace FlatZinc { out << "=====UNKNOWN=====" << std::endl; } } - delete sol; if (opt.mode() == SM_STAT) { double totalTime = (t_total.stop() / 1000.0); double solveTime = (t_solve.stop() / 1000.0); diff --git a/gecode/flatzinc/registry.cpp b/gecode/flatzinc/registry.cpp index 0d7bbaf6db..8df3f19214 100755 --- a/gecode/flatzinc/registry.cpp +++ b/gecode/flatzinc/registry.cpp @@ -1662,11 +1662,11 @@ namespace Gecode { namespace FlatZinc { } /// Read a `blackbox_exec` / `blackbox_dll` source annotation into \a mode, - /// \a instantiation (the executable/library) and \a args (its argument + /// \a target (the executable/library) and \a args (its argument /// list). Supports both the single-argument form (no arguments) and the /// `(target, args)` form. void blackbox_source(AST::Node* ann, std::string& mode, - std::string& instantiation, + std::string& target, std::vector& args) { auto string_arg = [](AST::Node* n, const char* what) { if ((n == nullptr) || !n->isString()) { @@ -1705,7 +1705,7 @@ namespace Gecode { namespace FlatZinc { "Registry", "Malformed blackbox annotation: expected a target string and " "an argument array."); } - instantiation = string_arg(arr->a[0], "target"); + target = string_arg(arr->a[0], "target"); if (!arr->a[1]->isArray()) { throw FlatZinc::Error( "Registry", "Malformed blackbox annotation: argument list must be an array " @@ -1716,15 +1716,15 @@ namespace Gecode { namespace FlatZinc { args.push_back(string_arg(al->a[i], "argument")); } } else { - instantiation = string_arg(c->args, "target"); + target = string_arg(c->args, "target"); } } void p_blackbox(FlatZincSpace& s, const ConExpr& ce, AST::Node* ann) { std::string mode; - std::string instantiation; + std::string target; std::vector args; - blackbox_source(ann, mode, instantiation, args); + blackbox_source(ann, mode, target, args); IntVarArgs int_input = s.arg2intvarargs(ce[0]); IntVarArgs int_output = s.arg2intvarargs(ce[2]); #ifdef GECODE_HAS_FLOAT_VARS @@ -1736,18 +1736,18 @@ namespace Gecode { namespace FlatZinc { "Blackbox propagator cannot use floating point values when Gecode is compiled without floating point decision variable support."); } #endif - FlatZinc::blackbox(s, BlackBoxAccess::state(s), int_input, int_output, + FlatZinc::blackbox(s, BlackBoxAccess::context(s), int_input, int_output, #ifdef GECODE_HAS_FLOAT_VARS float_input, float_output, #endif - mode, instantiation, args); + mode, target, args); } void p_blackbox_bounds(FlatZincSpace& s, const ConExpr& ce, AST::Node* ann) { std::string mode; - std::string instantiation; + std::string target; std::vector args; - blackbox_source(ann, mode, instantiation, args); + blackbox_source(ann, mode, target, args); IntVarArgs ivar = s.arg2intvarargs(ce[0]); #ifdef GECODE_HAS_FLOAT_VARS FloatVarArgs fvar = s.arg2floatvarargs(ce[1]); @@ -1762,11 +1762,11 @@ float_input, float_output, for (int i = 0; i < flat_reason.size(); i++) { reason[i] = flat_reason[i]; } - FlatZinc::blackbox_bounds(s, BlackBoxAccess::state(s), ivar, + FlatZinc::blackbox_bounds(s, BlackBoxAccess::context(s), ivar, #ifdef GECODE_HAS_FLOAT_VARS fvar, #endif - mode, instantiation, args, reason); + mode, target, args, reason); } class IntPoster { diff --git a/test/flatzinc/blackbox-exec.cpp b/test/flatzinc/blackbox-exec.cpp index 75d4d355e8..36f8c8f8fd 100644 --- a/test/flatzinc/blackbox-exec.cpp +++ b/test/flatzinc/blackbox-exec.cpp @@ -147,6 +147,21 @@ namespace { std::cout.flush(); } + bool + dependent_bounds(const std::string& request) { + long long xmin, xmax, ymin, ymax; + if (std::sscanf(request.c_str(), "%lld,%lld,%lld,%lld;", + &xmin, &xmax, &ymin, &ymax) != 4) { + return false; + } + (void) ymin; + (void) ymax; + write_response(std::to_string(xmin) + "," + std::to_string(xmax) + "," + + std::to_string(5 * xmin) + "," + + std::to_string(5 * xmax) + ";"); + return true; + } + const std::string* fixed_response(const std::string& mode) { static const std::string value7("7;"); @@ -248,6 +263,13 @@ main(int argc, char* argv[]) { continue; } + if (mode == "dependent_bounds") { + if (!dependent_bounds(request)) { + return 1; + } + continue; + } + if (const std::string* value = fixed_response(mode)) { write_response(*value); continue; diff --git a/test/flatzinc/blackbox.cpp b/test/flatzinc/blackbox.cpp index caa51f54bf..5eda7f5432 100644 --- a/test/flatzinc/blackbox.cpp +++ b/test/flatzinc/blackbox.cpp @@ -33,7 +33,7 @@ #include "test/flatzinc.hh" -#include +#include #include #include @@ -251,6 +251,30 @@ namespace Test { namespace FlatZinc { } namespace Blackbox { + class NativeProtocol : public Base { + public: + NativeProtocol(void) : Base("FlatZinc::blackbox::native_protocol") {} + virtual bool run(void) { + std::vector int_input{-2}; + std::vector float_input{1.25}; + std::vector int_output(1); + std::vector float_output(1); + Gecode::FlatZinc::BlackBoxCall call = { + int_input, float_input, int_output, float_output + }; + try { + if (Gecode::FlatZinc::encode_blackbox_request(call) != + "-2;1.25\n") { + return false; + } + Gecode::FlatZinc::decode_blackbox_response("7;2.5\n", call); + } catch (...) { + return false; + } + return (int_output[0] == 7) && (float_output[0] == 2.5); + } + }; + #ifdef GECODE_HAS_THREADS template bool @@ -344,6 +368,7 @@ namespace Test { namespace FlatZinc { public: /// Perform creation and registration Create(void) { + (void) new NativeProtocol; (void) new FlatZincErrorTest("blackbox::malformed_annotation", std::string(blackbox_decl) + "var 0..1: y;\n" @@ -396,6 +421,19 @@ namespace Test { namespace FlatZinc { "solve satisfy;\n", "x = 5;\n----------\n"); + (void) new FlatZincTest("blackbox::bounds_rescheduled_after_branch", + std::string(blackbox_bounds_decl) + + "var 0..1: x :: output_var;\n" + "var 0..5: y :: output_var;\n" + "constraint gecode_blackbox_bounds([x,y], [], " + "[1,0,0,2,1,1,1,1,1,2]) :: " + + fixture_annotation("exec", executable, {"dependent_bounds"}) + + ";\nsolve :: int_search([x], input_order, indomain_min, complete) " + "satisfy;\n", + "x = 0;\ny = 0;\n----------\n" + "x = 1;\ny = 5;\n----------\n==========\n", + false, {"-a"}); + #ifdef GECODE_HAS_FLOAT_VARS (void) new FlatZincErrorTest("blackbox::missing_bounds_reason_entry", std::string(blackbox_bounds_decl) + @@ -446,17 +484,6 @@ namespace Test { namespace FlatZinc { "satisfy;\n", {"-p", "2"}, "Failed to read output integer 0"); - (void) new FlatZincTest("blackbox::native_exec_rounds", - std::string(blackbox_decl) + - "var 1..1: a :: output_var;\n" - "var 2..2: b :: output_var;\n" - "constraint gecode_blackbox([], [], [a], []) :: " + - fixture_annotation("exec", executable, {"normal"}) + ";\n" - "constraint gecode_blackbox([], [], [b], []) :: " + - fixture_annotation("exec", executable, {"normal"}) + ";\n" - "solve satisfy;\n", - "a = 1;\nb = 2;\n----------\n"); - for (int kind = 1; kind <= 5; ++kind) { const char* expected = nullptr; switch (kind) { From fb5f0475f71fa8f1960d10eb81f95077e423de3f Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sun, 12 Jul 2026 18:28:36 +0200 Subject: [PATCH 10/14] Document FlatZinc blackbox contracts --- changelog.in | 4 +++- .../blackbox/blackbox_annotations.mzn | 5 ++++- .../blackbox/fzn_blackbox_bounds.mzn | 22 ++++++++++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/changelog.in b/changelog.in index 9f77bd4519..f945dc34a1 100755 --- a/changelog.in +++ b/changelog.in @@ -88,7 +88,9 @@ gecode_blackbox_bounds (bounds propagation, scheduled on bound changes). The blackbox_exec and blackbox_dll annotations select the execution mode and pass through extra arguments. These annotations intentionally execute user-provided code and should only be used with trusted models and trusted executable or -library paths. +library paths. Bounds callbacks must return valid enclosing intervals and give +complete dependency reasons; Gecode uses those reasons to decide when the +callback must run again. [ENTRY] Module: minimodel diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn index e929095667..88c9c1daf3 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn @@ -41,7 +41,10 @@ % ranges. Response lines are limited to 1 MiB. The helper is trusted code and % must keep reading requests and writing complete newline-terminated responses; % otherwise it can block the solver. During parallel search each worker thread -% gets its own process. +% gets its own process. Constraints with the same command and argument list +% share this worker-local process session. A blackbox_dll constraint instead +% owns its root instance; cloned spaces and search workers share that constraint +% backend according to the fzn_clone rules above. % POSIX exec teardown contains descendants that remain in the spawned process % group; helpers that deliberately detach are outside this containment and % trusted-code contract. diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn index cc79ad49b3..05044a0748 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn @@ -1,10 +1,22 @@ include "blackbox_annotations.mzn"; -% Bounds blackbox. The integer and float variables are encoded as lower/upper -% bound pairs, and the returned pairs narrow those same variables. flat_reason -% is the MiniZinc explanation encoding used to select which variable bounds can -% reschedule the blackbox. If it selects no dependencies, the blackbox is -% evaluated once at posting. +% Bounds blackbox. Integer variables come first, followed by float variables. +% Each array is encoded as consecutive lower/upper pairs, so variable i uses +% positions 2*i-1 and 2*i in the corresponding backend array. Returned bounds +% are intersected with the current domains; wider returned bounds therefore do +% not widen a Gecode variable. +% +% Every returned interval must contain every value that can occur in a solution +% compatible with the complete input box. Returning an invalid lower or upper +% bound can remove solutions and makes the model unsound. +% +% flat_reason contains one entry for every integer and float variable. Variable +% indices are 1-based over the combined list, with integer variables first. For +% each returned lower and upper bound, its reason must mention every input bound +% on which that result can depend. Gecode subscribes only to variables mentioned +% in these reasons. An omitted dependency can leave a stale result installed +% after an input bound changes and can make search unsound. If no reason mentions +% a dependency, the blackbox is evaluated once while posting. predicate fzn_blackbox_bounds( array[int] of var int: int_input, array[int] of var float: float_input, From 15a2675e9e94b35108e7fdfa59bd15c94bb4fcd3 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sun, 12 Jul 2026 20:31:12 +0200 Subject: [PATCH 11/14] Isolate blackbox process platforms --- CMakeLists.txt | 1 + Makefile.in | 5 +- cmake/GecodeSources.cmake | 3 + gecode/flatzinc/blackbox-backend.cpp | 885 +------------------ gecode/flatzinc/blackbox-backend.hh | 10 +- gecode/flatzinc/blackbox-process-none.cpp | 23 + gecode/flatzinc/blackbox-process-posix.cpp | 507 +++++++++++ gecode/flatzinc/blackbox-process-windows.cpp | 428 +++++++++ gecode/flatzinc/blackbox-process.hh | 54 ++ 9 files changed, 1038 insertions(+), 878 deletions(-) create mode 100644 gecode/flatzinc/blackbox-process-none.cpp create mode 100644 gecode/flatzinc/blackbox-process-posix.cpp create mode 100644 gecode/flatzinc/blackbox-process-windows.cpp create mode 100644 gecode/flatzinc/blackbox-process.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index b34a077e5c..8c347af670 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1631,6 +1631,7 @@ if(GECODE_INSTALL) PATTERN "mznlib" EXCLUDE PATTERN "flatzinc/blackbox.hh" EXCLUDE PATTERN "flatzinc/blackbox-backend.hh" EXCLUDE + PATTERN "flatzinc/blackbox-process.hh" EXCLUDE PATTERN "exampleplugin" EXCLUDE PATTERN "standalone-example" EXCLUDE PATTERN "abi*" EXCLUDE) diff --git a/Makefile.in b/Makefile.in index 7c317e632b..ac07d4e96b 100755 --- a/Makefile.in +++ b/Makefile.in @@ -822,12 +822,13 @@ endif # FLATZINCSRC0 = flatzinc.cpp registry.cpp branch.cpp blackbox-backend.cpp \ - blackbox-propagator.cpp + blackbox-process-none.cpp blackbox-process-posix.cpp \ + blackbox-process-windows.cpp blackbox-propagator.cpp FLATZINC_GENSRC0 = parser.tab.cpp lexer.yy.cpp FLATZINCHDR0 = ast.hh conexpr.hh option.hh parser.hh \ plugin.hh registry.hh symboltable.hh varspec.hh \ branch.hh branch.hpp lastval.hh complete.hh -FLATZINCPRIVATEHDR0 = blackbox.hh blackbox-backend.hh +FLATZINCPRIVATEHDR0 = blackbox.hh blackbox-backend.hh blackbox-process.hh FLATZINCSRC = $(FLATZINCSRC0:%=gecode/flatzinc/%) FLATZINC_GENSRC = $(FLATZINC_GENSRC0:%=gecode/flatzinc/%) diff --git a/cmake/GecodeSources.cmake b/cmake/GecodeSources.cmake index 8ef7efed43..40a2ffddd9 100644 --- a/cmake/GecodeSources.cmake +++ b/cmake/GecodeSources.cmake @@ -233,6 +233,9 @@ set(GECODE_GIST_SOURCES set(GECODE_FLATZINC_SOURCES gecode/flatzinc/blackbox-backend.cpp + gecode/flatzinc/blackbox-process-none.cpp + gecode/flatzinc/blackbox-process-posix.cpp + gecode/flatzinc/blackbox-process-windows.cpp gecode/flatzinc/blackbox-propagator.cpp gecode/flatzinc/branch.cpp gecode/flatzinc/flatzinc.cpp diff --git a/gecode/flatzinc/blackbox-backend.cpp b/gecode/flatzinc/blackbox-backend.cpp index 961c88e546..51a2a064f8 100644 --- a/gecode/flatzinc/blackbox-backend.cpp +++ b/gecode/flatzinc/blackbox-backend.cpp @@ -3,6 +3,9 @@ * Main authors: * Jip J. Dekker * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * * Copyright: * Jip J. Dekker, 2026 * @@ -48,6 +51,7 @@ #include #include #include +#include #include #include @@ -72,18 +76,6 @@ #include #else #include -#ifdef GECODE_HAS_POSIX_BLACKBOX_EXEC -#include -#include -#include -#include -#include -#include -#include -#include -#include -extern char **environ; -#endif #endif #ifdef GECODE_HAS_THREADS @@ -120,80 +112,6 @@ windows_error(const std::string &prefix, DWORD err) { return prefix + " (Windows error " + std::to_string(err) + ")"; } -class WindowsHandle { -private: - HANDLE handle; -public: - explicit WindowsHandle(HANDLE handle0=NULL) : handle(handle0) {} - ~WindowsHandle(void) { reset(); } - - WindowsHandle(const WindowsHandle &) = delete; - WindowsHandle &operator=(const WindowsHandle &) = delete; - - HANDLE get(void) const { return handle; } - HANDLE *put(void) { - reset(); - return &handle; - } - HANDLE release(void) { - HANDLE handle0 = handle; - handle = NULL; - return handle0; - } - bool valid(void) const { - return (handle != NULL) && (handle != INVALID_HANDLE_VALUE); - } - void reset(HANDLE handle0=NULL) { - if (valid()) { - CloseHandle(handle); - } - handle = handle0; - } -}; - -class WindowsAttributeList { -private: - std::vector buffer; - LPPROC_THREAD_ATTRIBUTE_LIST list; - bool initialized; -public: - WindowsAttributeList(void) : list(NULL), initialized(false) {} - ~WindowsAttributeList(void) { - if (initialized) { - DeleteProcThreadAttributeList(list); - } - } - - void init(void) { - SIZE_T size = 0; - InitializeProcThreadAttributeList(NULL, 1, 0, &size); - if (size == 0) { - throw Error("BlackBoxExec", - windows_error("ProcThreadAttributeList size query failed", - GetLastError())); - } - buffer.resize(size); - list = reinterpret_cast(buffer.data()); - if (!InitializeProcThreadAttributeList(list, 1, 0, &size)) { - throw Error("BlackBoxExec", - windows_error("InitializeProcThreadAttributeList failed", - GetLastError())); - } - initialized = true; - } - - void set_inherited_handles(HANDLE *handles, DWORD count) { - if (!UpdateProcThreadAttribute(list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, - handles, sizeof(HANDLE) * count, NULL, - NULL)) { - throw Error("BlackBoxExec", - windows_error("PROC_THREAD_ATTRIBUTE_HANDLE_LIST failed", - GetLastError())); - } - } - - LPPROC_THREAD_ATTRIBUTE_LIST get(void) const { return list; } -}; bool has_dll_suffix(const std::string &name) { @@ -233,11 +151,6 @@ dll_candidates(const std::string &name) { return candidates; } -bool -qualified_path(const std::wstring &program) { - return (program.find_first_of(L"\\/") != std::wstring::npos) || - ((program.size() > 1) && (program[1] == L':')); -} void close_library(void *library) { @@ -253,185 +166,8 @@ close_library(void *library) { } } -#ifdef GECODE_HAS_POSIX_BLACKBOX_EXEC -int -set_cloexec(int fd) { - int flags = fcntl(fd, F_GETFD); - if (flags == -1) { - return -1; - } - return fcntl(fd, F_SETFD, flags | FD_CLOEXEC); -} - -int -dup_cloexec(int fd, int min_fd) { - int nfd; -#ifdef F_DUPFD_CLOEXEC - nfd = fcntl(fd, F_DUPFD_CLOEXEC, min_fd); - if (nfd != -1) { - return nfd; - } - if (errno != EINVAL) { - return -1; - } #endif - nfd = fcntl(fd, F_DUPFD, min_fd); - if (nfd == -1) { - return -1; - } - if (set_cloexec(nfd) != 0) { - int e = errno; - ::close(nfd); - errno = e; - return -1; - } - return nfd; -} - -int -move_from_standard_fd(int fd) { - if (fd > STDERR_FILENO) { - return fd; - } - int nfd = dup_cloexec(fd, STDERR_FILENO + 1); - if (nfd == -1) { - return -1; - } - ::close(fd); - return nfd; -} - -class FileDescriptor { -private: - int fd; -public: - explicit FileDescriptor(int fd0=-1) : fd(fd0) {} - ~FileDescriptor(void) { reset(); } - - int get(void) const { return fd; } - int release(void) { - int fd0 = fd; - fd = -1; - return fd0; - } - void reset(int fd0=-1) { - if (fd != -1) { - ::close(fd); - } - fd = fd0; - } -}; - -int -move_away_from_standard_fd(FileDescriptor &fd) { - int old = fd.release(); - int nfd = move_from_standard_fd(old); - if (nfd == -1) { - fd.reset(old); - } else { - fd.reset(nfd); - } - return nfd; -} -class SpawnFileActions { -private: - posix_spawn_file_actions_t actions; - bool initialized; -public: - SpawnFileActions(void) : initialized(false) {} - ~SpawnFileActions(void) { - if (initialized) { - posix_spawn_file_actions_destroy(&actions); - } - } - - int init(void) { - int err = posix_spawn_file_actions_init(&actions); - initialized = err == 0; - return err; - } - posix_spawn_file_actions_t *get(void) { return &actions; } -}; - -class SpawnAttributes { -private: - posix_spawnattr_t attr; - bool initialized; -public: - SpawnAttributes(void) : initialized(false) {} - ~SpawnAttributes(void) { - if (initialized) { - posix_spawnattr_destroy(&attr); - } - } - - int init(void) { - int err = posix_spawnattr_init(&attr); - initialized = err == 0; - return err; - } - posix_spawnattr_t *get(void) { return &attr; } -}; - -int -create_socketpair(int sv[2]) { -#ifdef SOCK_CLOEXEC - if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == 0) { - return 0; - } - if (errno != EINVAL) { - return -1; - } -#endif - if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) { - return -1; - } - if ((set_cloexec(sv[0]) != 0) || (set_cloexec(sv[1]) != 0)) { - int e = errno; - ::close(sv[0]); - ::close(sv[1]); - errno = e; - return -1; - } - return 0; -} - -ssize_t -send_no_sigpipe(int fd, const char *data, size_t size) { -#ifdef MSG_NOSIGNAL - return send(fd, data, size, MSG_NOSIGNAL); -#else -#ifdef SO_NOSIGPIPE - return send(fd, data, size, 0); -#else - sigset_t block; - sigset_t old; - sigset_t pending; - sigemptyset(&block); - sigaddset(&block, SIGPIPE); - bool blocked = false; - bool was_pending = false; - if (pthread_sigmask(SIG_BLOCK, &block, &old) == 0) { - blocked = true; - if (sigpending(&pending) == 0) { - was_pending = sigismember(&pending, SIGPIPE) == 1; - } - } - ssize_t n = send(fd, data, size, 0); - if ((n == -1) && (errno == EPIPE) && !was_pending) { - const struct timespec timeout = {0, 0}; - sigtimedwait(&block, NULL, &timeout); - } - if (blocked) { - pthread_sigmask(SIG_SETMASK, &old, NULL); - } - return n; -#endif -#endif -} -#endif -#endif template T @@ -501,8 +237,6 @@ check_floats(const std::vector &v, const char *source) { } #endif -const size_t max_exec_response_size = 1024 * 1024; - } // namespace #ifdef GECODE_HAS_THREADS @@ -701,603 +435,6 @@ BlackBoxLibrary::run(BlackBoxCall& call) { #endif } -class BlackBoxExec::Session { -protected: -#ifdef _WIN32 - HANDLE job; - HANDLE process; - HANDLE pipe_send; - HANDLE pipe_receive; -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) - pid_t child; - int pipe_send; - FILE *file_receive; -#endif -#ifdef GECODE_HAS_THREADS - std::thread::id owner; -#endif - - static std::string last_error(const std::string &prefix) { -#ifdef _WIN32 - return prefix + " (Windows error " + std::to_string(GetLastError()) + ")"; -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) - return prefix + " (errno " + std::to_string(errno) + ")"; -#else - return prefix; -#endif - } - -#ifdef _WIN32 - static void close_handle(HANDLE &h) { - if (h != NULL) { - CloseHandle(h); - h = NULL; - } - } - - static std::wstring quote_argument(const std::wstring &arg) { - std::wstring q(L"\""); - unsigned int backslashes = 0; - for (wchar_t ch : arg) { - if (ch == L'\\') { - backslashes++; - } else if (ch == L'"') { - q.append(backslashes * 2 + 1, L'\\'); - q += ch; - backslashes = 0; - } else { - q.append(backslashes, L'\\'); - q += ch; - backslashes = 0; - } - } - q.append(backslashes * 2, L'\\'); - q += L'"'; - return q; - } - - void open_windows(const std::string &program, - const std::vector &args); - void close_windows(void); -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) - static void sleep_grace_period(void) { - struct timespec remaining = {0, 10000000}; - while ((nanosleep(&remaining, &remaining) == -1) && (errno == EINTR)) {} - } - - static bool child_exited(pid_t pid) { - siginfo_t info; - do { - info.si_pid = 0; - if (waitid(P_PID, pid, &info, WEXITED | WNOHANG | WNOWAIT) == 0) { - return info.si_pid != 0; - } - } while (errno == EINTR); - return false; - } - - static void signal_group(pid_t pid, int signal) { - if ((kill(-pid, signal) == -1) && (errno == ESRCH)) { - return; - } - } - - static void wait_group(pid_t pid, int attempts) { - for (int i = 0; i < attempts; i++) { - if ((kill(-pid, 0) == -1) && (errno == ESRCH)) { - return; - } - if (child_exited(pid)) { - return; - } - sleep_grace_period(); - } - } - - static void terminate_child(pid_t pid) { - if (pid <= 0) { - return; - } - int status = 0; - // Keep the child unreaped until the group has received both signals. - signal_group(pid, SIGTERM); - wait_group(pid, 100); - signal_group(pid, SIGKILL); - do { - if (waitpid(pid, &status, 0) != -1) { - return; - } - } while (errno == EINTR); - } - - static void check_sigchld(void) { - struct sigaction action; - if (sigaction(SIGCHLD, NULL, &action) != 0) { - throw Error("BlackBoxExec", last_error("SIGCHLD query failed")); - } - if ((action.sa_handler != SIG_DFL) -#ifdef SA_NOCLDWAIT - || (action.sa_flags & SA_NOCLDWAIT) -#endif - ) { - throw Error("BlackBoxExec", - "Cannot start a blackbox process unless SIGCHLD uses " - "SIG_DFL without SA_NOCLDWAIT"); - } - } - - void open_posix(const std::string &program, - const std::vector &args); - void close_posix(void); -#endif - -public: - Session(const std::string &program, const std::vector &args) -#ifdef _WIN32 - : job(NULL), process(NULL), pipe_send(NULL), pipe_receive(NULL) -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) - : child(-1), pipe_send(-1), file_receive(NULL) -#endif - { -#ifdef GECODE_HAS_THREADS - owner = std::this_thread::get_id(); -#endif -#ifdef _WIN32 - open_windows(program, args); -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) - open_posix(program, args); -#else - (void)program; - (void)args; - throw Error("BlackBoxExec", - "Persistent process blackboxes are not supported on this " - "platform"); -#endif - } - - ~Session(void) { close(); } - - bool owned_by_current_thread(void) const { -#ifdef GECODE_HAS_THREADS - return owner == std::this_thread::get_id(); -#else - return true; -#endif - } - - std::string run(const std::string &out_buf) { -#ifdef _WIN32 - size_t written = 0; - while (written < out_buf.size()) { - DWORD count = 0; - DWORD remaining = - static_cast(out_buf.size() - written); - BOOL success = - WriteFile(pipe_send, out_buf.data() + written, remaining, &count, - nullptr); - if (!success || count == 0) { - throw Error("BlackBoxExec", - last_error("Writing blackbox process input failed")); - } - written += count; - } - - char c[2] = {0, 0}; - std::ostringstream oss; - size_t response_size = 0; - while (c[0] != '\n') { - DWORD count = 0; - BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); - if (!success) { - if (GetLastError() == ERROR_BROKEN_PIPE) { - throw Error("BlackBoxExec", - "Blackbox process provided an incomplete response"); - } - throw Error( - "BlackBoxExec", - "Failed to read blackbox process output from pipe"); - } else if (count == 0) { - throw Error("BlackBoxExec", - "Blackbox process provided an incomplete response"); - } - assert(count == 1); - if (++response_size > max_exec_response_size) { - throw Error("BlackBoxExec", - "Blackbox process response exceeds the size limit"); - } - oss << c[0]; - } - return oss.str(); -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) - const char *p = out_buf.c_str(); - size_t remaining = out_buf.size(); - while (remaining > 0) { - ssize_t n = send_no_sigpipe(pipe_send, p, remaining); - if (n < 0) { - if (errno == EINTR) { - continue; - } - throw Error("BlackBoxExec", - "Writing blackbox process input failed with errno " + - std::to_string(errno)); - } - if (n == 0) { - throw Error("BlackBoxExec", - "Writing blackbox process input wrote zero bytes"); - } - p += n; - remaining -= static_cast(n); - } - - std::string in_buffer; - while (true) { - errno = 0; - int ch = fgetc(file_receive); - if (ch == EOF) { - if (feof(file_receive)) { - throw Error("BlackBoxExec", - "Blackbox process provided an incomplete response"); - } - int err = errno; - if (err == EINTR) { - clearerr(file_receive); - continue; - } - throw Error("BlackBoxExec", - std::string("Reading blackbox process output from pipe " - "failed with errno ") + - std::to_string(err)); - } - in_buffer += static_cast(ch); - if (in_buffer.size() > max_exec_response_size) { - throw Error("BlackBoxExec", - "Blackbox process response exceeds the size limit"); - } - if (ch == '\n') { - break; - } - } - return in_buffer; -#else - (void)out_buf; - throw Error("BlackBoxExec", - "Persistent process blackboxes are not supported on this " - "platform"); -#endif - } - - void close(void) { -#ifdef _WIN32 - close_windows(); -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) - close_posix(); -#endif - } -}; - -#ifdef _WIN32 -void -BlackBoxExec::Session::open_windows(const std::string &program, - const std::vector &args) { - // Build the command line before opening OS handles so allocation/conversion - // failures cannot leak partially constructed process state. - std::wstring program_w = utf8_to_wide(program); - std::wstring prog = quote_argument(program_w); - for (const std::string &a : args) { - prog += L" "; - prog += quote_argument(utf8_to_wide(a)); - } - std::vector cmdline(prog.begin(), prog.end()); - cmdline.push_back(L'\0'); - - SECURITY_ATTRIBUTES saAttr; - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - saAttr.lpSecurityDescriptor = NULL; - - WindowsHandle child_stdin_read; - WindowsHandle child_stdin_write; - WindowsHandle child_stdout_read; - WindowsHandle child_stdout_write; - WindowsHandle child_stderr_write; - if (!CreatePipe(child_stdout_read.put(), child_stdout_write.put(), &saAttr, - 0)) { - throw Error("BlackBoxExec", last_error("Stdout CreatePipe failed")); - } - if (!SetHandleInformation(child_stdout_read.get(), HANDLE_FLAG_INHERIT, 0)) { - throw Error("BlackBoxExec", - last_error("Stdout SetHandleInformation failed")); - } - if (!CreatePipe(child_stdin_read.put(), child_stdin_write.put(), &saAttr, - 0)) { - throw Error("BlackBoxExec", last_error("Stdin CreatePipe failed")); - } - if (!SetHandleInformation(child_stdin_write.get(), HANDLE_FLAG_INHERIT, 0)) { - throw Error("BlackBoxExec", - last_error("Stdin SetHandleInformation failed")); - } - - HANDLE parent_stderr = GetStdHandle(STD_ERROR_HANDLE); - if ((parent_stderr != NULL) && (parent_stderr != INVALID_HANDLE_VALUE)) { - if (!DuplicateHandle(GetCurrentProcess(), parent_stderr, - GetCurrentProcess(), child_stderr_write.put(), 0, TRUE, - DUPLICATE_SAME_ACCESS)) { - throw Error("BlackBoxExec", - last_error("stderr DuplicateHandle failed")); - } - } else { - HANDLE nul = CreateFileW(L"NUL", GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, &saAttr, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (nul == INVALID_HANDLE_VALUE) { - throw Error("BlackBoxExec", last_error("stderr NUL CreateFile failed")); - } - child_stderr_write.reset(nul); - } - - WindowsAttributeList attr_list; - attr_list.init(); - PROCESS_INFORMATION piProcInfo; - STARTUPINFOEXW siStartInfo; - ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFOEXW)); - siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEXW); - siStartInfo.StartupInfo.hStdOutput = child_stdout_write.get(); - siStartInfo.StartupInfo.hStdInput = child_stdin_read.get(); - siStartInfo.StartupInfo.hStdError = child_stderr_write.get(); - siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; - - HANDLE inherit_handles[3] = {child_stdin_read.get(), child_stdout_write.get(), - child_stderr_write.get()}; - attr_list.set_inherited_handles(inherit_handles, 3); - siStartInfo.lpAttributeList = attr_list.get(); - - WindowsHandle process_job(CreateJobObjectW(NULL, NULL)); - if (!process_job.valid()) { - throw Error("BlackBoxExec", last_error("CreateJobObject failed")); - } - JOBOBJECT_EXTENDED_LIMIT_INFORMATION job_info; - ZeroMemory(&job_info, sizeof(job_info)); - job_info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if (!SetInformationJobObject(process_job.get(), - JobObjectExtendedLimitInformation, &job_info, - sizeof(job_info))) { - throw Error("BlackBoxExec", last_error("SetInformationJobObject failed")); - } - - BOOL processStarted = - CreateProcessW(qualified_path(program_w) ? program_w.c_str() : NULL, - cmdline.data(), // command line - nullptr, // process security attributes - nullptr, // primary thread security attributes - TRUE, // handles from attribute list - EXTENDED_STARTUPINFO_PRESENT | CREATE_SUSPENDED, - nullptr, // use parent's environment - nullptr, // use parent's current directory - &siStartInfo.StartupInfo, - &piProcInfo); // receives PROCESS_INFORMATION - - if (!processStarted) { - throw Error("BlackBoxExec", - windows_error("starting blackbox process failed for program `" + - program + "'", GetLastError())); - } - WindowsHandle process_handle(piProcInfo.hProcess); - WindowsHandle thread_handle(piProcInfo.hThread); - if (!AssignProcessToJobObject(process_job.get(), process_handle.get())) { - DWORD err = GetLastError(); - DWORD terminate_err = ERROR_SUCCESS; - if (!TerminateProcess(process_handle.get(), 1)) { - terminate_err = GetLastError(); - } - DWORD wait = WaitForSingleObject(process_handle.get(), 5000); - std::string message = windows_error( - "Unable to assign blackbox process to required job", err); - if (terminate_err != ERROR_SUCCESS) { - message += "; " + windows_error("TerminateProcess cleanup failed", - terminate_err); - } - if (wait == WAIT_FAILED) { - message += "; " + last_error("process cleanup wait failed"); - } else if (wait == WAIT_TIMEOUT) { - message += "; process cleanup timed out"; - } - throw Error("BlackBoxExec", message); - } - - if (ResumeThread(thread_handle.get()) == static_cast(-1)) { - DWORD err = GetLastError(); - DWORD terminate_err = ERROR_SUCCESS; - if (!TerminateJobObject(process_job.get(), 1)) { - terminate_err = GetLastError(); - } - HANDLE assigned_job = process_job.release(); - DWORD close_err = ERROR_SUCCESS; - if (!CloseHandle(assigned_job)) { - close_err = GetLastError(); - } - DWORD wait = WaitForSingleObject(process_handle.get(), 5000); - std::string message = windows_error( - "ResumeThread failed for blackbox process", err); - if (terminate_err != ERROR_SUCCESS) { - message += "; " + windows_error("TerminateJobObject cleanup failed", - terminate_err); - } - if (close_err != ERROR_SUCCESS) { - message += "; " + windows_error("job cleanup close failed", close_err); - } - if (wait == WAIT_FAILED) { - message += "; " + last_error("process cleanup wait failed"); - } else if (wait == WAIT_TIMEOUT) { - message += "; process cleanup timed out"; - } - throw Error("BlackBoxExec", message); - } - - pipe_send = child_stdin_write.release(); - pipe_receive = child_stdout_read.release(); - process = process_handle.release(); - job = process_job.release(); -} - -void -BlackBoxExec::Session::close_windows(void) { - close_handle(pipe_send); - close_handle(pipe_receive); - if (process != NULL) { - DWORD wait = WaitForSingleObject(process, 1000); - if (wait == WAIT_TIMEOUT) { - if (job != NULL) { - TerminateJobObject(job, 1); - } else { - TerminateProcess(process, 1); - } - WaitForSingleObject(process, 5000); - } - close_handle(process); - } - close_handle(job); -} -#elif defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) -void -BlackBoxExec::Session::open_posix(const std::string &program, - const std::vector &args) { - const int READ = 0; - const int WRITE = 1; - - std::vector argv; - argv.reserve(args.size() + 2); - argv.push_back(const_cast(program.c_str())); - for (const std::string &a : args) { - argv.push_back(const_cast(a.c_str())); - } - argv.push_back(nullptr); - - check_sigchld(); - - FileDescriptor child_in[2]; - FileDescriptor child_out[2]; - int fds[2]; - if (create_socketpair(fds) != 0) { - throw Error("BlackBoxExec", last_error("stdin socket creation failed")); - } - child_in[READ].reset(fds[READ]); - child_in[WRITE].reset(fds[WRITE]); - if (create_socketpair(fds) != 0) { - throw Error("BlackBoxExec", last_error("stdout socket creation failed")); - } - child_out[READ].reset(fds[READ]); - child_out[WRITE].reset(fds[WRITE]); - FileDescriptor *session_fds[] = { - &child_in[READ], &child_in[WRITE], - &child_out[READ], &child_out[WRITE] - }; - for (FileDescriptor *fd : session_fds) { - if (move_away_from_standard_fd(*fd) == -1) { - throw Error("BlackBoxExec", - last_error("moving session descriptors away from stdio " - "failed")); - } - } - - SpawnFileActions actions; - int err = actions.init(); - if (err != 0) { - errno = err; - throw Error("BlackBoxExec", last_error("spawn file action init failed")); - } - - SpawnAttributes attr; - err = attr.init(); - if (err != 0) { - errno = err; - throw Error("BlackBoxExec", last_error("spawn attribute init failed")); - } - - err = posix_spawnattr_setpgroup(attr.get(), 0); - if (err == 0) { - short flags = POSIX_SPAWN_SETPGROUP; -#if defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ - defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP) - flags |= POSIX_SPAWN_CLOEXEC_DEFAULT; -#endif - err = posix_spawnattr_setflags(attr.get(), flags); - } - if (err == 0) { - err = posix_spawn_file_actions_adddup2(actions.get(), - child_in[READ].get(), - STDIN_FILENO); - } - if (err == 0) { - err = posix_spawn_file_actions_adddup2(actions.get(), - child_out[WRITE].get(), - STDOUT_FILENO); - } -#if defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ - defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP) - if (err == 0) { - err = posix_spawn_file_actions_addinherit_np(actions.get(), - STDERR_FILENO); - } -#elif defined(GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP) - if (err == 0) { - err = posix_spawn_file_actions_addclosefrom_np(actions.get(), - STDERR_FILENO + 1); - } -#endif - if (err == 0) { - err = posix_spawnp(&child, program.c_str(), actions.get(), attr.get(), - argv.data(), environ); - } - if (err != 0) { - child = -1; - errno = err; - throw Error("BlackBoxExec", last_error("starting blackbox process failed")); - } - - child_in[READ].reset(); - child_out[WRITE].reset(); - -#ifdef SO_NOSIGPIPE - int nosigpipe = 1; - if (setsockopt(child_in[WRITE].get(), SOL_SOCKET, SO_NOSIGPIPE, &nosigpipe, - sizeof(nosigpipe)) != 0) { - int e = errno; - terminate_child(child); - child = -1; - errno = e; - throw Error("BlackBoxExec", last_error("SO_NOSIGPIPE setup failed")); - } -#endif - FILE *receive = fdopen(child_out[READ].get(), "r"); - if (receive == NULL) { - int e = errno; - terminate_child(child); - child = -1; - errno = e; - throw Error("BlackBoxExec", last_error("fdopen failed")); - } - file_receive = receive; - child_out[READ].release(); - pipe_send = child_in[WRITE].release(); -} - -void -BlackBoxExec::Session::close_posix(void) { - if (pipe_send != -1) { - ::close(pipe_send); - pipe_send = -1; - } - if (file_receive != NULL) { - fclose(file_receive); - file_receive = NULL; - } - if (child > 0) { - terminate_child(child); - child = -1; - } -} -#endif BlackBoxExec::BlackBoxExec(const std::string &program0, const std::vector &args0) @@ -1305,22 +442,23 @@ BlackBoxExec::BlackBoxExec(const std::string &program0, BlackBoxExec::~BlackBoxExec(void) { Support::Lock lock(mutex); - for (Session *s : sessions) { + for (BlackBoxProcessSession *s : sessions) { delete s; } sessions.clear(); } -BlackBoxExec::Session &BlackBoxExec::session(void) { +BlackBoxProcessSession& BlackBoxExec::session(void) { Support::Lock lock(mutex); - for (Session *s : sessions) { + for (BlackBoxProcessSession *s : sessions) { if (s->owned_by_current_thread()) { return *s; } } - std::unique_ptr s(new Session(program, args)); - Session *r = s.get(); + std::unique_ptr s( + create_blackbox_process(program, args)); + BlackBoxProcessSession *r = s.get(); sessions.push_back(r); s.release(); return *r; @@ -1468,7 +606,8 @@ decode_blackbox_response(const std::string& response, BlackBoxCall& call) { void BlackBoxExec::run(BlackBoxCall& call) { - const std::string response = session().run(encode_blackbox_request(call)); + const std::string response = + session().exchange(encode_blackbox_request(call)); decode_blackbox_response(response, call); } diff --git a/gecode/flatzinc/blackbox-backend.hh b/gecode/flatzinc/blackbox-backend.hh index b08f97c1ca..e37537d9c2 100644 --- a/gecode/flatzinc/blackbox-backend.hh +++ b/gecode/flatzinc/blackbox-backend.hh @@ -3,6 +3,9 @@ * Main authors: * Jip J. Dekker * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * * Copyright: * Jip J. Dekker, 2026 * @@ -54,6 +57,8 @@ namespace Gecode { namespace FlatZinc { +class BlackBoxProcessSession; + /// Inputs and pre-sized output buffers for one backend call. struct BlackBoxCall { const std::vector& int_input; @@ -113,12 +118,11 @@ public: void run(BlackBoxCall& call) override; protected: - class Session; std::string program; std::vector args; Support::Mutex mutex; - std::vector sessions; - Session& session(void); + std::vector sessions; + BlackBoxProcessSession& session(void); }; /// Encode one request for the executable backend's line protocol. diff --git a/gecode/flatzinc/blackbox-process-none.cpp b/gecode/flatzinc/blackbox-process-none.cpp new file mode 100644 index 0000000000..7c1317cb8f --- /dev/null +++ b/gecode/flatzinc/blackbox-process-none.cpp @@ -0,0 +1,23 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Contributing authors: + * Mikael Zayenz Lagerkvist + * + * Copyright: + * Jip J. Dekker, 2026 + */ +#include +#include + +#if !defined(_WIN32) && !defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) +namespace Gecode { namespace FlatZinc { + +BlackBoxProcessSession* +create_blackbox_process(const std::string&, const std::vector&) { + throw Error("BlackBoxExec", + "Persistent process blackboxes are not supported on this " + "platform"); +} + +}} +#endif diff --git a/gecode/flatzinc/blackbox-process-posix.cpp b/gecode/flatzinc/blackbox-process-posix.cpp new file mode 100644 index 0000000000..c5a466966c --- /dev/null +++ b/gecode/flatzinc/blackbox-process-posix.cpp @@ -0,0 +1,507 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * + * Copyright: + * Jip J. Dekker, 2026 + */ +#include +#include + +#if defined(GECODE_HAS_POSIX_BLACKBOX_EXEC) && !defined(_WIN32) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +namespace Gecode { namespace FlatZinc { +namespace { + +const size_t max_exec_response_size = 1024 * 1024; + +int +set_cloexec(int fd) { + int flags = fcntl(fd, F_GETFD); + if (flags == -1) { + return -1; + } + return fcntl(fd, F_SETFD, flags | FD_CLOEXEC); +} + +int +dup_cloexec(int fd, int min_fd) { + int nfd; +#ifdef F_DUPFD_CLOEXEC + nfd = fcntl(fd, F_DUPFD_CLOEXEC, min_fd); + if (nfd != -1) { + return nfd; + } + if (errno != EINVAL) { + return -1; + } +#endif + nfd = fcntl(fd, F_DUPFD, min_fd); + if (nfd == -1) { + return -1; + } + if (set_cloexec(nfd) != 0) { + int e = errno; + ::close(nfd); + errno = e; + return -1; + } + return nfd; +} + +int +move_from_standard_fd(int fd) { + if (fd > STDERR_FILENO) { + return fd; + } + int nfd = dup_cloexec(fd, STDERR_FILENO + 1); + if (nfd == -1) { + return -1; + } + ::close(fd); + return nfd; +} + +class FileDescriptor { +private: + int fd; +public: + explicit FileDescriptor(int fd0=-1) : fd(fd0) {} + ~FileDescriptor(void) { reset(); } + + int get(void) const { return fd; } + int release(void) { + int fd0 = fd; + fd = -1; + return fd0; + } + void reset(int fd0=-1) { + if (fd != -1) { + ::close(fd); + } + fd = fd0; + } +}; + +int +move_away_from_standard_fd(FileDescriptor &fd) { + int old = fd.release(); + int nfd = move_from_standard_fd(old); + if (nfd == -1) { + fd.reset(old); + } else { + fd.reset(nfd); + } + return nfd; +} + +class SpawnFileActions { +private: + posix_spawn_file_actions_t actions; + bool initialized; +public: + SpawnFileActions(void) : initialized(false) {} + ~SpawnFileActions(void) { + if (initialized) { + posix_spawn_file_actions_destroy(&actions); + } + } + + int init(void) { + int err = posix_spawn_file_actions_init(&actions); + initialized = err == 0; + return err; + } + posix_spawn_file_actions_t *get(void) { return &actions; } +}; + +class SpawnAttributes { +private: + posix_spawnattr_t attr; + bool initialized; +public: + SpawnAttributes(void) : initialized(false) {} + ~SpawnAttributes(void) { + if (initialized) { + posix_spawnattr_destroy(&attr); + } + } + + int init(void) { + int err = posix_spawnattr_init(&attr); + initialized = err == 0; + return err; + } + posix_spawnattr_t *get(void) { return &attr; } +}; + +int +create_socketpair(int sv[2]) { +#ifdef SOCK_CLOEXEC + if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == 0) { + return 0; + } + if (errno != EINVAL) { + return -1; + } +#endif + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) { + return -1; + } + if ((set_cloexec(sv[0]) != 0) || (set_cloexec(sv[1]) != 0)) { + int e = errno; + ::close(sv[0]); + ::close(sv[1]); + errno = e; + return -1; + } + return 0; +} + +ssize_t +send_no_sigpipe(int fd, const char *data, size_t size) { +#ifdef MSG_NOSIGNAL + return send(fd, data, size, MSG_NOSIGNAL); +#else +#ifdef SO_NOSIGPIPE + return send(fd, data, size, 0); +#else + sigset_t block; + sigset_t old; + sigset_t pending; + sigemptyset(&block); + sigaddset(&block, SIGPIPE); + bool blocked = false; + bool was_pending = false; + if (pthread_sigmask(SIG_BLOCK, &block, &old) == 0) { + blocked = true; + if (sigpending(&pending) == 0) { + was_pending = sigismember(&pending, SIGPIPE) == 1; + } + } + ssize_t n = send(fd, data, size, 0); + if ((n == -1) && (errno == EPIPE) && !was_pending) { + const struct timespec timeout = {0, 0}; + sigtimedwait(&block, NULL, &timeout); + } + if (blocked) { + pthread_sigmask(SIG_SETMASK, &old, NULL); + } + return n; +#endif +#endif +} +class PosixProcessSession : public BlackBoxProcessSession { +protected: + pid_t child; + int pipe_send; + FILE *file_receive; + + static std::string last_error(const std::string &prefix) { + return prefix + " (errno " + std::to_string(errno) + ")"; + } + + static void sleep_grace_period(void) { + struct timespec remaining = {0, 10000000}; + while ((nanosleep(&remaining, &remaining) == -1) && (errno == EINTR)) {} + } + + static bool child_exited(pid_t pid) { + siginfo_t info; + do { + info.si_pid = 0; + if (waitid(P_PID, pid, &info, WEXITED | WNOHANG | WNOWAIT) == 0) { + return info.si_pid != 0; + } + } while (errno == EINTR); + return false; + } + + static void signal_group(pid_t pid, int signal) { + if ((kill(-pid, signal) == -1) && (errno == ESRCH)) { + return; + } + } + + static void wait_group(pid_t pid, int attempts) { + for (int i = 0; i < attempts; i++) { + if ((kill(-pid, 0) == -1) && (errno == ESRCH)) { + return; + } + if (child_exited(pid)) { + return; + } + sleep_grace_period(); + } + } + + static void terminate_child(pid_t pid) { + if (pid <= 0) { + return; + } + int status = 0; + // Keep the child unreaped until the group has received both signals. + signal_group(pid, SIGTERM); + wait_group(pid, 100); + signal_group(pid, SIGKILL); + do { + if (waitpid(pid, &status, 0) != -1) { + return; + } + } while (errno == EINTR); + } + + static void check_sigchld(void) { + struct sigaction action; + if (sigaction(SIGCHLD, NULL, &action) != 0) { + throw Error("BlackBoxExec", last_error("SIGCHLD query failed")); + } + if ((action.sa_handler != SIG_DFL) +#ifdef SA_NOCLDWAIT + || (action.sa_flags & SA_NOCLDWAIT) +#endif + ) { + throw Error("BlackBoxExec", + "Cannot start a blackbox process unless SIGCHLD uses " + "SIG_DFL without SA_NOCLDWAIT"); + } + } + + void open_posix(const std::string &program, + const std::vector &args); + void close_posix(void); + +public: + PosixProcessSession(const std::string &program, const std::vector &args) + : child(-1), pipe_send(-1), file_receive(NULL) + { + open_posix(program, args); + } + + ~PosixProcessSession(void) { close(); } + + std::string exchange(const std::string &out_buf) { + const char *p = out_buf.c_str(); + size_t remaining = out_buf.size(); + while (remaining > 0) { + ssize_t n = send_no_sigpipe(pipe_send, p, remaining); + if (n < 0) { + if (errno == EINTR) { + continue; + } + throw Error("BlackBoxExec", + "Writing blackbox process input failed with errno " + + std::to_string(errno)); + } + if (n == 0) { + throw Error("BlackBoxExec", + "Writing blackbox process input wrote zero bytes"); + } + p += n; + remaining -= static_cast(n); + } + + std::string in_buffer; + while (true) { + errno = 0; + int ch = fgetc(file_receive); + if (ch == EOF) { + if (feof(file_receive)) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + int err = errno; + if (err == EINTR) { + clearerr(file_receive); + continue; + } + throw Error("BlackBoxExec", + std::string("Reading blackbox process output from pipe " + "failed with errno ") + + std::to_string(err)); + } + in_buffer += static_cast(ch); + if (in_buffer.size() > max_exec_response_size) { + throw Error("BlackBoxExec", + "Blackbox process response exceeds the size limit"); + } + if (ch == '\n') { + break; + } + } + return in_buffer; + } + + void close(void) { + close_posix(); + } +}; + +void +PosixProcessSession::open_posix(const std::string& program, + const std::vector& args) { + const int READ = 0; + const int WRITE = 1; + + std::vector argv; + argv.reserve(args.size() + 2); + argv.push_back(const_cast(program.c_str())); + for (const std::string &a : args) { + argv.push_back(const_cast(a.c_str())); + } + argv.push_back(nullptr); + + check_sigchld(); + + FileDescriptor child_in[2]; + FileDescriptor child_out[2]; + int fds[2]; + if (create_socketpair(fds) != 0) { + throw Error("BlackBoxExec", last_error("stdin socket creation failed")); + } + child_in[READ].reset(fds[READ]); + child_in[WRITE].reset(fds[WRITE]); + if (create_socketpair(fds) != 0) { + throw Error("BlackBoxExec", last_error("stdout socket creation failed")); + } + child_out[READ].reset(fds[READ]); + child_out[WRITE].reset(fds[WRITE]); + FileDescriptor *session_fds[] = { + &child_in[READ], &child_in[WRITE], + &child_out[READ], &child_out[WRITE] + }; + for (FileDescriptor *fd : session_fds) { + if (move_away_from_standard_fd(*fd) == -1) { + throw Error("BlackBoxExec", + last_error("moving session descriptors away from stdio " + "failed")); + } + } + + SpawnFileActions actions; + int err = actions.init(); + if (err != 0) { + errno = err; + throw Error("BlackBoxExec", last_error("spawn file action init failed")); + } + + SpawnAttributes attr; + err = attr.init(); + if (err != 0) { + errno = err; + throw Error("BlackBoxExec", last_error("spawn attribute init failed")); + } + + err = posix_spawnattr_setpgroup(attr.get(), 0); + if (err == 0) { + short flags = POSIX_SPAWN_SETPGROUP; +#if defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ + defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP) + flags |= POSIX_SPAWN_CLOEXEC_DEFAULT; +#endif + err = posix_spawnattr_setflags(attr.get(), flags); + } + if (err == 0) { + err = posix_spawn_file_actions_adddup2(actions.get(), + child_in[READ].get(), + STDIN_FILENO); + } + if (err == 0) { + err = posix_spawn_file_actions_adddup2(actions.get(), + child_out[WRITE].get(), + STDOUT_FILENO); + } +#if defined(GECODE_HAS_POSIX_SPAWN_CLOEXEC_DEFAULT) && \ + defined(GECODE_HAS_POSIX_SPAWN_ADDINHERIT_NP) + if (err == 0) { + err = posix_spawn_file_actions_addinherit_np(actions.get(), + STDERR_FILENO); + } +#elif defined(GECODE_HAS_POSIX_SPAWN_ADDCLOSEFROM_NP) + if (err == 0) { + err = posix_spawn_file_actions_addclosefrom_np(actions.get(), + STDERR_FILENO + 1); + } +#endif + if (err == 0) { + err = posix_spawnp(&child, program.c_str(), actions.get(), attr.get(), + argv.data(), environ); + } + if (err != 0) { + child = -1; + errno = err; + throw Error("BlackBoxExec", last_error("starting blackbox process failed")); + } + + child_in[READ].reset(); + child_out[WRITE].reset(); + +#ifdef SO_NOSIGPIPE + int nosigpipe = 1; + if (setsockopt(child_in[WRITE].get(), SOL_SOCKET, SO_NOSIGPIPE, &nosigpipe, + sizeof(nosigpipe)) != 0) { + int e = errno; + terminate_child(child); + child = -1; + errno = e; + throw Error("BlackBoxExec", last_error("SO_NOSIGPIPE setup failed")); + } +#endif + FILE *receive = fdopen(child_out[READ].get(), "r"); + if (receive == NULL) { + int e = errno; + terminate_child(child); + child = -1; + errno = e; + throw Error("BlackBoxExec", last_error("fdopen failed")); + } + file_receive = receive; + child_out[READ].release(); + pipe_send = child_in[WRITE].release(); +} + +void +PosixProcessSession::close_posix(void) { + if (pipe_send != -1) { + ::close(pipe_send); + pipe_send = -1; + } + if (file_receive != NULL) { + fclose(file_receive); + file_receive = NULL; + } + if (child > 0) { + terminate_child(child); + child = -1; + } +} + +} // namespace + +BlackBoxProcessSession* +create_blackbox_process(const std::string& program, + const std::vector& args) { + return new PosixProcessSession(program, args); +} + +}} +#endif diff --git a/gecode/flatzinc/blackbox-process-windows.cpp b/gecode/flatzinc/blackbox-process-windows.cpp new file mode 100644 index 0000000000..fc1cbaff1a --- /dev/null +++ b/gecode/flatzinc/blackbox-process-windows.cpp @@ -0,0 +1,428 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * + * Copyright: + * Jip J. Dekker, 2026 + */ +#if defined(_WIN32) +#if !defined(_WIN32_WINNT) || (_WIN32_WINNT < 0x0600) +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 +#endif +#if !defined(WINVER) || (WINVER < 0x0600) +#undef WINVER +#define WINVER 0x0600 +#endif + +#include +#include +#include +#include +#include + +namespace Gecode { namespace FlatZinc { +namespace { + +const size_t max_exec_response_size = 1024 * 1024; + +std::wstring +utf8_to_wide(const std::string &s) { + if (s.empty()) { + return std::wstring(); + } + int n = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.c_str(), + static_cast(s.size()), NULL, 0); + if (n == 0) { + throw Error("Blackbox", "Invalid UTF-8 string in blackbox path or argument"); + } + std::wstring w(static_cast(n), L'\0'); + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.c_str(), + static_cast(s.size()), &w[0], n) == 0) { + throw Error("Blackbox", "Invalid UTF-8 string in blackbox path or argument"); + } + return w; +} + +std::string +windows_error(const std::string &prefix, DWORD err) { + return prefix + " (Windows error " + std::to_string(err) + ")"; +} + +class WindowsHandle { +private: + HANDLE handle; +public: + explicit WindowsHandle(HANDLE handle0=NULL) : handle(handle0) {} + ~WindowsHandle(void) { reset(); } + + WindowsHandle(const WindowsHandle &) = delete; + WindowsHandle &operator=(const WindowsHandle &) = delete; + + HANDLE get(void) const { return handle; } + HANDLE *put(void) { + reset(); + return &handle; + } + HANDLE release(void) { + HANDLE handle0 = handle; + handle = NULL; + return handle0; + } + bool valid(void) const { + return (handle != NULL) && (handle != INVALID_HANDLE_VALUE); + } + void reset(HANDLE handle0=NULL) { + if (valid()) { + CloseHandle(handle); + } + handle = handle0; + } +}; + +class WindowsAttributeList { +private: + std::vector buffer; + LPPROC_THREAD_ATTRIBUTE_LIST list; + bool initialized; +public: + WindowsAttributeList(void) : list(NULL), initialized(false) {} + ~WindowsAttributeList(void) { + if (initialized) { + DeleteProcThreadAttributeList(list); + } + } + + void init(void) { + SIZE_T size = 0; + InitializeProcThreadAttributeList(NULL, 1, 0, &size); + if (size == 0) { + throw Error("BlackBoxExec", + windows_error("ProcThreadAttributeList size query failed", + GetLastError())); + } + buffer.resize(size); + list = reinterpret_cast(buffer.data()); + if (!InitializeProcThreadAttributeList(list, 1, 0, &size)) { + throw Error("BlackBoxExec", + windows_error("InitializeProcThreadAttributeList failed", + GetLastError())); + } + initialized = true; + } + + void set_inherited_handles(HANDLE *handles, DWORD count) { + if (!UpdateProcThreadAttribute(list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + handles, sizeof(HANDLE) * count, NULL, + NULL)) { + throw Error("BlackBoxExec", + windows_error("PROC_THREAD_ATTRIBUTE_HANDLE_LIST failed", + GetLastError())); + } + } + + LPPROC_THREAD_ATTRIBUTE_LIST get(void) const { return list; } +}; +bool +qualified_path(const std::wstring &program) { + return (program.find_first_of(L"\\/") != std::wstring::npos) || + ((program.size() > 1) && (program[1] == L':')); +} + +class WindowsProcessSession : public BlackBoxProcessSession { +protected: + HANDLE job; + HANDLE process; + HANDLE pipe_send; + HANDLE pipe_receive; + + static std::string last_error(const std::string &prefix) { + return prefix + " (Windows error " + std::to_string(GetLastError()) + ")"; + } + + static void close_handle(HANDLE &h) { + if (h != NULL) { + CloseHandle(h); + h = NULL; + } + } + + static std::wstring quote_argument(const std::wstring &arg) { + std::wstring q(L"\""); + unsigned int backslashes = 0; + for (wchar_t ch : arg) { + if (ch == L'\\') { + backslashes++; + } else if (ch == L'"') { + q.append(backslashes * 2 + 1, L'\\'); + q += ch; + backslashes = 0; + } else { + q.append(backslashes, L'\\'); + q += ch; + backslashes = 0; + } + } + q.append(backslashes * 2, L'\\'); + q += L'"'; + return q; + } + + void open_windows(const std::string &program, + const std::vector &args); + void close_windows(void); + +public: + WindowsProcessSession(const std::string &program, const std::vector &args) + : job(NULL), process(NULL), pipe_send(NULL), pipe_receive(NULL) + { + open_windows(program, args); + } + + ~WindowsProcessSession(void) { close(); } + + std::string exchange(const std::string &out_buf) { + size_t written = 0; + while (written < out_buf.size()) { + DWORD count = 0; + DWORD remaining = + static_cast(out_buf.size() - written); + BOOL success = + WriteFile(pipe_send, out_buf.data() + written, remaining, &count, + nullptr); + if (!success || count == 0) { + throw Error("BlackBoxExec", + last_error("Writing blackbox process input failed")); + } + written += count; + } + + char c[2] = {0, 0}; + std::ostringstream oss; + size_t response_size = 0; + while (c[0] != '\n') { + DWORD count = 0; + BOOL success = ReadFile(pipe_receive, c, sizeof(c) - 1, &count, NULL); + if (!success) { + if (GetLastError() == ERROR_BROKEN_PIPE) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + throw Error( + "BlackBoxExec", + "Failed to read blackbox process output from pipe"); + } else if (count == 0) { + throw Error("BlackBoxExec", + "Blackbox process provided an incomplete response"); + } + assert(count == 1); + if (++response_size > max_exec_response_size) { + throw Error("BlackBoxExec", + "Blackbox process response exceeds the size limit"); + } + oss << c[0]; + } + return oss.str(); + } + + void close(void) { + close_windows(); + } +}; + +void +WindowsProcessSession::open_windows(const std::string &program, + const std::vector &args) { + // Build the command line before opening OS handles so allocation/conversion + // failures cannot leak partially constructed process state. + std::wstring program_w = utf8_to_wide(program); + std::wstring prog = quote_argument(program_w); + for (const std::string &a : args) { + prog += L" "; + prog += quote_argument(utf8_to_wide(a)); + } + std::vector cmdline(prog.begin(), prog.end()); + cmdline.push_back(L'\0'); + + SECURITY_ATTRIBUTES saAttr; + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + + WindowsHandle child_stdin_read; + WindowsHandle child_stdin_write; + WindowsHandle child_stdout_read; + WindowsHandle child_stdout_write; + WindowsHandle child_stderr_write; + if (!CreatePipe(child_stdout_read.put(), child_stdout_write.put(), &saAttr, + 0)) { + throw Error("BlackBoxExec", last_error("Stdout CreatePipe failed")); + } + if (!SetHandleInformation(child_stdout_read.get(), HANDLE_FLAG_INHERIT, 0)) { + throw Error("BlackBoxExec", + last_error("Stdout SetHandleInformation failed")); + } + if (!CreatePipe(child_stdin_read.put(), child_stdin_write.put(), &saAttr, + 0)) { + throw Error("BlackBoxExec", last_error("Stdin CreatePipe failed")); + } + if (!SetHandleInformation(child_stdin_write.get(), HANDLE_FLAG_INHERIT, 0)) { + throw Error("BlackBoxExec", + last_error("Stdin SetHandleInformation failed")); + } + + HANDLE parent_stderr = GetStdHandle(STD_ERROR_HANDLE); + if ((parent_stderr != NULL) && (parent_stderr != INVALID_HANDLE_VALUE)) { + if (!DuplicateHandle(GetCurrentProcess(), parent_stderr, + GetCurrentProcess(), child_stderr_write.put(), 0, TRUE, + DUPLICATE_SAME_ACCESS)) { + throw Error("BlackBoxExec", + last_error("stderr DuplicateHandle failed")); + } + } else { + HANDLE nul = CreateFileW(L"NUL", GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, &saAttr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (nul == INVALID_HANDLE_VALUE) { + throw Error("BlackBoxExec", last_error("stderr NUL CreateFile failed")); + } + child_stderr_write.reset(nul); + } + + WindowsAttributeList attr_list; + attr_list.init(); + PROCESS_INFORMATION piProcInfo; + STARTUPINFOEXW siStartInfo; + ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); + ZeroMemory(&siStartInfo, sizeof(STARTUPINFOEXW)); + siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEXW); + siStartInfo.StartupInfo.hStdOutput = child_stdout_write.get(); + siStartInfo.StartupInfo.hStdInput = child_stdin_read.get(); + siStartInfo.StartupInfo.hStdError = child_stderr_write.get(); + siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; + + HANDLE inherit_handles[3] = {child_stdin_read.get(), child_stdout_write.get(), + child_stderr_write.get()}; + attr_list.set_inherited_handles(inherit_handles, 3); + siStartInfo.lpAttributeList = attr_list.get(); + + WindowsHandle process_job(CreateJobObjectW(NULL, NULL)); + if (!process_job.valid()) { + throw Error("BlackBoxExec", last_error("CreateJobObject failed")); + } + JOBOBJECT_EXTENDED_LIMIT_INFORMATION job_info; + ZeroMemory(&job_info, sizeof(job_info)); + job_info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (!SetInformationJobObject(process_job.get(), + JobObjectExtendedLimitInformation, &job_info, + sizeof(job_info))) { + throw Error("BlackBoxExec", last_error("SetInformationJobObject failed")); + } + + BOOL processStarted = + CreateProcessW(qualified_path(program_w) ? program_w.c_str() : NULL, + cmdline.data(), // command line + nullptr, // process security attributes + nullptr, // primary thread security attributes + TRUE, // handles from attribute list + EXTENDED_STARTUPINFO_PRESENT | CREATE_SUSPENDED, + nullptr, // use parent's environment + nullptr, // use parent's current directory + &siStartInfo.StartupInfo, + &piProcInfo); // receives PROCESS_INFORMATION + + if (!processStarted) { + throw Error("BlackBoxExec", + windows_error("starting blackbox process failed for program `" + + program + "'", GetLastError())); + } + WindowsHandle process_handle(piProcInfo.hProcess); + WindowsHandle thread_handle(piProcInfo.hThread); + if (!AssignProcessToJobObject(process_job.get(), process_handle.get())) { + DWORD err = GetLastError(); + DWORD terminate_err = ERROR_SUCCESS; + if (!TerminateProcess(process_handle.get(), 1)) { + terminate_err = GetLastError(); + } + DWORD wait = WaitForSingleObject(process_handle.get(), 5000); + std::string message = windows_error( + "Unable to assign blackbox process to required job", err); + if (terminate_err != ERROR_SUCCESS) { + message += "; " + windows_error("TerminateProcess cleanup failed", + terminate_err); + } + if (wait == WAIT_FAILED) { + message += "; " + last_error("process cleanup wait failed"); + } else if (wait == WAIT_TIMEOUT) { + message += "; process cleanup timed out"; + } + throw Error("BlackBoxExec", message); + } + + if (ResumeThread(thread_handle.get()) == static_cast(-1)) { + DWORD err = GetLastError(); + DWORD terminate_err = ERROR_SUCCESS; + if (!TerminateJobObject(process_job.get(), 1)) { + terminate_err = GetLastError(); + } + HANDLE assigned_job = process_job.release(); + DWORD close_err = ERROR_SUCCESS; + if (!CloseHandle(assigned_job)) { + close_err = GetLastError(); + } + DWORD wait = WaitForSingleObject(process_handle.get(), 5000); + std::string message = windows_error( + "ResumeThread failed for blackbox process", err); + if (terminate_err != ERROR_SUCCESS) { + message += "; " + windows_error("TerminateJobObject cleanup failed", + terminate_err); + } + if (close_err != ERROR_SUCCESS) { + message += "; " + windows_error("job cleanup close failed", close_err); + } + if (wait == WAIT_FAILED) { + message += "; " + last_error("process cleanup wait failed"); + } else if (wait == WAIT_TIMEOUT) { + message += "; process cleanup timed out"; + } + throw Error("BlackBoxExec", message); + } + + pipe_send = child_stdin_write.release(); + pipe_receive = child_stdout_read.release(); + process = process_handle.release(); + job = process_job.release(); +} + +void +WindowsProcessSession::close_windows(void) { + close_handle(pipe_send); + close_handle(pipe_receive); + if (process != NULL) { + DWORD wait = WaitForSingleObject(process, 1000); + if (wait == WAIT_TIMEOUT) { + if (job != NULL) { + TerminateJobObject(job, 1); + } else { + TerminateProcess(process, 1); + } + WaitForSingleObject(process, 5000); + } + close_handle(process); + } + close_handle(job); +} +} // namespace + +BlackBoxProcessSession* +create_blackbox_process(const std::string& program, + const std::vector& args) { + return new WindowsProcessSession(program, args); +} + +}} +#endif diff --git a/gecode/flatzinc/blackbox-process.hh b/gecode/flatzinc/blackbox-process.hh new file mode 100644 index 0000000000..2c90477a9e --- /dev/null +++ b/gecode/flatzinc/blackbox-process.hh @@ -0,0 +1,54 @@ +/* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ +/* + * Main authors: + * Jip J. Dekker + * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * + * Copyright: + * Jip J. Dekker, 2026 + */ +#ifndef GECODE_FLATZINC_BLACKBOX_PROCESS_HH +#define GECODE_FLATZINC_BLACKBOX_PROCESS_HH + +#include +#include + +#ifdef GECODE_HAS_THREADS +#include +#endif + +namespace Gecode { namespace FlatZinc { + +/// Platform process session used by the executable blackbox backend. +class BlackBoxProcessSession { +protected: +#ifdef GECODE_HAS_THREADS + std::thread::id owner; +#endif + BlackBoxProcessSession(void) +#ifdef GECODE_HAS_THREADS + : owner(std::this_thread::get_id()) +#endif + {} +public: + virtual ~BlackBoxProcessSession(void) {} + bool owned_by_current_thread(void) const { +#ifdef GECODE_HAS_THREADS + return owner == std::this_thread::get_id(); +#else + return true; +#endif + } + virtual std::string exchange(const std::string& request) = 0; +}; + +/// Create the process implementation selected for the target platform. +BlackBoxProcessSession* +create_blackbox_process(const std::string& program, + const std::vector& args); + +}} + +#endif From 20ccaa7e5bda6e73fcaaad154636a071f4b40b6c Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sun, 12 Jul 2026 20:31:17 +0200 Subject: [PATCH 12/14] Credit FlatZinc blackbox contributors --- changelog.in | 20 +++++++++----------- gecode/flatzinc/blackbox-propagator.cpp | 3 +++ gecode/flatzinc/blackbox.hh | 3 +++ test/flatzinc/blackbox-dll.cpp | 3 +++ test/flatzinc/blackbox-exec.cpp | 3 +++ test/flatzinc/blackbox.cpp | 3 +++ 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/changelog.in b/changelog.in index f945dc34a1..a97b2d6660 100755 --- a/changelog.in +++ b/changelog.in @@ -79,18 +79,16 @@ autoconf build path, and updates CI coverage for current platforms. Module: flatzinc What: new Rank: minor +Thanks: Jip J. Dekker [DESCRIPTION] -Add support for the experimental MiniZinc black-box propagator interface. A -FlatZinc model can request propagation using an external function, implemented -either as a shared library or as a subprocess, through two generic propagators: -gecode_blackbox (value propagation, scheduled once all inputs are fixed) and -gecode_blackbox_bounds (bounds propagation, scheduled on bound changes). The -blackbox_exec and blackbox_dll annotations select the execution mode and pass -through extra arguments. These annotations intentionally execute user-provided -code and should only be used with trusted models and trusted executable or -library paths. Bounds callbacks must return valid enclosing intervals and give -complete dependency reasons; Gecode uses those reasons to decide when the -callback must run again. +Add support for the experimental MiniZinc black-box propagator interface, +contributed by Jip J. Dekker. +[MORE] +FlatZinc models can implement custom value and bounds propagation in a shared +library or persistent subprocess. The blackbox_dll and blackbox_exec annotations +select the backend and pass its arguments. Bounds callbacks return enclosing +intervals together with the dependencies that determine when the callback must +run again. [ENTRY] Module: minimodel diff --git a/gecode/flatzinc/blackbox-propagator.cpp b/gecode/flatzinc/blackbox-propagator.cpp index 28a0fb1e64..7177871221 100644 --- a/gecode/flatzinc/blackbox-propagator.cpp +++ b/gecode/flatzinc/blackbox-propagator.cpp @@ -3,6 +3,9 @@ * Main authors: * Jip J. Dekker * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * * Copyright: * Jip J. Dekker, 2026 * diff --git a/gecode/flatzinc/blackbox.hh b/gecode/flatzinc/blackbox.hh index c8cbbb83fa..99285430e0 100644 --- a/gecode/flatzinc/blackbox.hh +++ b/gecode/flatzinc/blackbox.hh @@ -3,6 +3,9 @@ * Main authors: * Jip J. Dekker * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * * Copyright: * Jip J. Dekker, 2026 * diff --git a/test/flatzinc/blackbox-dll.cpp b/test/flatzinc/blackbox-dll.cpp index 74ef4a770f..ebab86261b 100644 --- a/test/flatzinc/blackbox-dll.cpp +++ b/test/flatzinc/blackbox-dll.cpp @@ -1,5 +1,8 @@ /* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ /* + * Contributing authors: + * Mikael Zayenz Lagerkvist + * * Copyright: * Jip J. Dekker, 2026 * diff --git a/test/flatzinc/blackbox-exec.cpp b/test/flatzinc/blackbox-exec.cpp index 36f8c8f8fd..350fc2c193 100644 --- a/test/flatzinc/blackbox-exec.cpp +++ b/test/flatzinc/blackbox-exec.cpp @@ -1,5 +1,8 @@ /* -*- mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- */ /* + * Contributing authors: + * Mikael Zayenz Lagerkvist + * * Copyright: * Jip J. Dekker, 2026 * diff --git a/test/flatzinc/blackbox.cpp b/test/flatzinc/blackbox.cpp index 5eda7f5432..9f6acbe696 100644 --- a/test/flatzinc/blackbox.cpp +++ b/test/flatzinc/blackbox.cpp @@ -3,6 +3,9 @@ * Main authors: * Jip J. Dekker * + * Contributing authors: + * Mikael Zayenz Lagerkvist + * * Copyright: * Jip J. Dekker, 2026 * From 0bb9fd98c9da2789bd0ced68aa1dfea85304f37d Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sun, 12 Jul 2026 23:09:10 +0200 Subject: [PATCH 13/14] Use MiniZinc blackbox annotations --- .../blackbox/blackbox_annotations.mzn | 59 ------------------- .../experimental/blackbox/fzn_blackbox.mzn | 2 - .../blackbox/fzn_blackbox_bounds.mzn | 2 - 3 files changed, 63 deletions(-) delete mode 100644 gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn deleted file mode 100644 index 88c9c1daf3..0000000000 --- a/gecode/flatzinc/mznlib/experimental/blackbox/blackbox_annotations.mzn +++ /dev/null @@ -1,59 +0,0 @@ -% Blackbox annotations execute user-provided code. Use them only for trusted -% models and trusted executable/library paths. -% -% blackbox_dll loads a native library with the following C ABI: -% -% extern "C" { -% void* fzn_init(const char** args, size_t n_args); -% void* fzn_clone(void* instance); -% void fzn_blackbox( -% void* instance, -% const int64_t* int_in, size_t n_int_in, -% const double* float_in, size_t n_float_in, -% int64_t* int_out, size_t n_int_out, -% double* float_out, size_t n_float_out); -% void fzn_free(void* instance); -% } -% -% fzn_blackbox is required. fzn_init, fzn_clone, and fzn_free are optional; -% exporting fzn_init requires exporting fzn_clone. If fzn_init is absent, the -% library is stateless: fzn_blackbox receives NULL and can be called -% concurrently. If fzn_free is absent, Gecode does not release library state. -% -% Gecode calls fzn_init once for each blackbox_dll constraint, with the -% annotation arguments. In threaded builds its result is a prototype: each -% calling thread receives and reuses one fzn_clone result, and calls using the -% same clone are serialized. Different clones can be called concurrently. In -% builds without threads, fzn_blackbox receives the fzn_init result directly. -% When fzn_free is exported, Gecode calls it once for every fzn_init and -% fzn_clone result: once for the root and once for each clone. -% -% On Windows, export all functions with __declspec(dllexport) and use -% __stdcall. On other systems, use the ordinary C calling convention. -% -% blackbox_exec starts a persistent subprocess. For each call, Gecode writes one -% line to stdin: -% -% comma-separated integer inputs ; comma-separated float inputs -% -% The process must answer with one line in the same format for the expected -% integer and float outputs. Outputs must be finite and in Gecode's numeric -% ranges. Response lines are limited to 1 MiB. The helper is trusted code and -% must keep reading requests and writing complete newline-terminated responses; -% otherwise it can block the solver. During parallel search each worker thread -% gets its own process. Constraints with the same command and argument list -% share this worker-local process session. A blackbox_dll constraint instead -% owns its root instance; cloned spaces and search workers share that constraint -% backend according to the fzn_clone rules above. -% POSIX exec teardown contains descendants that remain in the spawned process -% group; helpers that deliberately detach are outside this containment and -% trusted-code contract. -% -% All blackboxes must be deterministic from FlatZinc's point of view: equal -% input arrays must produce equal output arrays, regardless of call order or -% internal instance-state evolution. -annotation blackbox_dll(string: library); -annotation blackbox_dll(string: library, array[int] of string: args); - -annotation blackbox_exec(string: command); -annotation blackbox_exec(string: command, array[int] of string: args); diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn index 5706f7ad73..c5cbc7a853 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox.mzn @@ -1,5 +1,3 @@ -include "blackbox_annotations.mzn"; - % Value blackbox. The output arrays are constrained to the values returned by % the selected blackbox when all input variables are fixed. If both input arrays % are empty, the blackbox is evaluated once at posting. diff --git a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn index 05044a0748..30d0498b39 100644 --- a/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn +++ b/gecode/flatzinc/mznlib/experimental/blackbox/fzn_blackbox_bounds.mzn @@ -1,5 +1,3 @@ -include "blackbox_annotations.mzn"; - % Bounds blackbox. Integer variables come first, followed by float variables. % Each array is encoded as consecutive lower/upper pairs, so variable i uses % positions 2*i-1 and 2*i in the corresponding backend array. Returned bounds From 517652f0f5a3859e4ff90f896d610212336eb338 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Mon, 13 Jul 2026 08:27:27 +0200 Subject: [PATCH 14/14] Remove redundant blackbox instance lock --- gecode/flatzinc/blackbox-backend.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gecode/flatzinc/blackbox-backend.cpp b/gecode/flatzinc/blackbox-backend.cpp index 51a2a064f8..a1dfd427d7 100644 --- a/gecode/flatzinc/blackbox-backend.cpp +++ b/gecode/flatzinc/blackbox-backend.cpp @@ -244,7 +244,6 @@ class BlackBoxLibrary::Instance { public: std::thread::id owner; void *value; - Support::Mutex mutex; Instance(const std::thread::id &owner0, void *value0) : owner(owner0), value(value0) {} @@ -379,6 +378,7 @@ BlackBoxLibrary::Instance * BlackBoxLibrary::instance(void) { const std::thread::id owner = std::this_thread::get_id(); Support::Lock lock(mutex); + // Each live thread has exclusive access to its selected instance. for (Instance *instance : instances) { if (instance->owner == owner) { return instance; @@ -407,7 +407,6 @@ BlackBoxLibrary::run(BlackBoxCall& call) { #ifdef GECODE_HAS_THREADS if (library_fzn_init != nullptr) { Instance *selected = this->instance(); - Support::Lock lock(selected->mutex); library_fzn_blackbox(selected->value, call.int_input.data(), call.int_input.size(), call.float_input.data(), call.float_input.size(),