Bug: Hidden attributes not permitted by declaration regex

[CBroz1]

Bug Report

Description

According to documentation, adding a _ prefix to a table field will make it hidden. Is this feature intended for user defined tables?

Reproducibility

Include:

• OS: Linux
• Python Version: 3.13.13
• MySQL Version: 8.0.33
• MySQL Deployment Strategy: local docker
• DataJoint Version: 2.2.0
• Minimum number of steps to reliably reproduce the issue
• Complete error stack as a result of evaluating the above steps

Example code to reproduce the issue

import sys

import datajoint as dj

schema = dj.Schema("test_hidden")

print(f"DataJoint version: {dj.__version__}")
print(f"Python version: {sys.version}")


@schema
class MyTable(dj.Lookup):
    definition = """
    id: int
    ---
    _hidden: bool
    """

    contents = [
        (1, False),
        (2, True),
    ]

Error stack

: run temp-test-hidden.py
DataJoint version: 2.2.0
Python version: 3.13.13 | packaged by Anaconda, Inc. | (main, Apr 14 2026, 06:19:41) [GCC 14.3.0]
[2026-04-15 17:04:30][WARNING]: Native type 'int' is used in attribute 'id'. Consider using a core DataJoint type for better portability.
---------------------------------------------------------------------------
ParseException                            Traceback (most recent call last)
File ~/wrk/alt/dj2/src/datajoint/declare.py:859, in compile_attribute(line, in_key, foreign_key_sql, context, adapter)
    858 try:
--> 859     match = attribute_parser.parse_string(line + "#", parse_all=True)
    860 except pp.ParseException as err:

File ~/miniconda3/envs/dj2/lib/python3.13/site-packages/pyparsing/core.py:1346, in ParserElement.parse_string(self, instring, parse_all, **kwargs)
   1345     # catch and re-raise exception from here, clearing out pyparsing internal stack trace
-> 1346     raise exc.with_traceback(None)
   1347 else:

ParseException: Expected W:(a-z, 0-9_a-z), found '_'  (at char 0), (line:1, col:1)

During handling of the above exception, another exception occurred:

DataJointError                            Traceback (most recent call last)
File ~/wrk/alt/dj2/temp-test-hidden.py:11
      7 print(f"DataJoint version: {dj.__version__}")
      8 print(f"Python version: {sys.version}")
---> 11 @schema
     12 class MyTable(dj.Lookup):
     13     definition = """
     14     id: int
     15     ---
     16     _hidden: bool
     17     """
     19     contents = [
     20         (1, False),
     21         (2, True),
     22     ]

File ~/wrk/alt/dj2/src/datajoint/schemas.py:235, in _Schema.__call__(self, cls, context)
    233     raise DataJointError("The schema decorator should not be applied to Part tables.")
    234 if self.is_activated():
--> 235     self._decorate_master(cls, context)
    236 else:
    237     self.declare_list.append((cls, context))

File ~/wrk/alt/dj2/src/datajoint/schemas.py:251, in _Schema._decorate_master(self, cls, context)
    240 def _decorate_master(self, cls: type, context: dict[str, Any]) -> None:
    241     """
    242     Process a master table class and its part tables.
    243
   (...)    249         Declaration context for foreign key resolution.
    250     """
--> 251     self._decorate_table(cls, context=dict(context, self=cls, **{cls.__name__: cls}))
    252     # Process part tables
    253     for part in ordered_dir(cls):

File ~/wrk/alt/dj2/src/datajoint/schemas.py:297, in _Schema._decorate_table(self, table_class, context, assert_declared)
    293 create_tables = (
    294     self.create_tables if self.create_tables is not None else self.connection._config.database.create_tables
    295 )
    296 if not is_declared and not assert_declared and create_tables:
--> 297     instance.declare(context)
    298     self.connection.dependencies.clear()
    299 is_declared = is_declared or instance.is_declared

File ~/wrk/alt/dj2/src/datajoint/table.py:154, in Table.declare(self, context)
    149 if not is_camel_case(class_name):
    150     raise DataJointError(
    151         f"Table class name `{self.class_name}` is invalid. "
    152         "Class names must be in CamelCase, starting with a capital letter."
    153     )
--> 154 sql, _external_stores, primary_key, fk_attribute_map, pre_ddl, post_ddl = declare(
    155     self.full_table_name, self.definition, context, self.connection.adapter, config=self.connection._config
    156 )
    158 # Call declaration hook for validation (subclasses like AutoPopulate can override)
    159 self._declare_check(primary_key, fk_attribute_map)

File ~/wrk/alt/dj2/src/datajoint/declare.py:453, in declare(full_table_name, definition, context, adapter, config)
    437 if len(table_name) > adapter.max_table_name_length:
    438     raise DataJointError(
    439         "Table name `{name}` exceeds the max length of {max_length}".format(
    440             name=table_name, max_length=adapter.max_table_name_length
    441         )
    442     )
    444 (
    445     table_comment,
    446     primary_key,
    447     attribute_sql,
    448     foreign_key_sql,
    449     index_sql,
    450     external_stores,
    451     fk_attribute_map,
    452     column_comments,
--> 453 ) = prepare_declare(definition, context, adapter)
    455 # Add hidden job metadata for Computed/Imported tables (not parts)
    456 if config is None:

File ~/wrk/alt/dj2/src/datajoint/declare.py:375, in prepare_declare(definition, context, adapter)
    373     compile_index(re.sub(r"\s*#.*$", "", line), index_sql, adapter)
    374 else:
--> 375     name, sql, store, comment = compile_attribute(line, in_key, foreign_key_sql, context, adapter)
    376     if store:
    377         external_stores.append(store)

File ~/wrk/alt/dj2/src/datajoint/declare.py:861, in compile_attribute(line, in_key, foreign_key_sql, context, adapter)
    859     match = attribute_parser.parse_string(line + "#", parse_all=True)
    860 except pp.ParseException as err:
--> 861     raise DataJointError(
    862         "Declaration error in position {pos} in line:\n  {line}\n{msg}".format(
    863             line=err.args[0], pos=err.args[1], msg=err.args[2]
    864         )
    865     )
    866 match["comment"] = match["comment"].rstrip("#")
    867 if "default" not in match:

DataJointError: Declaration error in position 0 in line:
  _hidden: bool#
Expected W:(a-z, 0-9_a-z)

Expected Behavior

A user should be able to define a _-prefixed field for hiding. My goal usecase
is a hash of a json field to be stored as a hidden unique index.

@schema
class MyParams(dj.Manual):
    definition = """
    id: int
    ---
    tool: varchar(32)
    params: json
    _params_hash: varchar(32)
    unique index (tool, _params_hash)
    """