From 2fdebf9ec1a7ccd01766bf7698425d608289acdb Mon Sep 17 00:00:00 2001 From: Komorebi623 Date: Tue, 18 Aug 2026 19:48:23 +0800 Subject: [PATCH 1/3] fix(minimax-h3): align Ref2VA audio and conditioning paths - read prompt files before generation when no inline prompt is supplied - encode MiniMax-H3 reference audio per stereo channel and resample standalone audio consistently - build Ref2VA presentation as ordered reference segments for image/video/audio inputs - share condition noise RNG across reference latents and add debug dumps for alignment checks - enforce a minimum MiniMax-H3 generation canvas to avoid invalid tiny outputs - document MiniMax-H3 output resolution constraints --- docs/minimax-h3.md | 10 + examples/cli/main.cpp | 19 ++ .../autoencoders/minimax_h3_audio_vae.hpp | 21 +- .../components/text_encoders/llm.hpp | 57 +++++- src/dit_models/models/minimax_h3_full.hpp | 9 + .../pipelines/minimax_h3_pipeline.cpp | 182 ++++++++++++++---- 6 files changed, 246 insertions(+), 52 deletions(-) diff --git a/docs/minimax-h3.md b/docs/minimax-h3.md index ab6d883..1890517 100644 --- a/docs/minimax-h3.md +++ b/docs/minimax-h3.md @@ -127,6 +127,16 @@ seconds. The resolved value is printed before generation. Use `--video-frames ` when an exact legal frame count is required. The two options are mutually exclusive. +## Output resolution + +Output width and height must be multiples of 32. MiniMax-H3 is trained and +released for 768p base generation; the official workflow uses a 768-pixel +short edge by default. edge-dit.cpp additionally rejects output canvases below +65,536 pixels (for example, `128x128`) because the model can produce +structurally invalid block mosaics at those extremely small sizes. Use at least +`256x256`; use a 768-pixel short edge for the model's recommended quality +range. + ## Usage Set component paths for the desired precision. The video and audio VAE files are diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 0fc3b69..5b977e7 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -871,6 +872,24 @@ int main(int argc, char** argv) { return launch_status; } + std::string prompt_file_contents; + if ((args.prompt == nullptr || args.prompt[0] == '\0') && + args.prompt_file != nullptr && args.prompt_file[0] != '\0') { + std::ifstream prompt_file(args.prompt_file, std::ios::binary); + if (!prompt_file.is_open()) { + std::fprintf(stderr, "failed to open prompt file: %s\n", args.prompt_file); + return 1; + } + std::ostringstream prompt_stream; + prompt_stream << prompt_file.rdbuf(); + prompt_file_contents = prompt_stream.str(); + if (prompt_file_contents.empty()) { + std::fprintf(stderr, "prompt file is empty: %s\n", args.prompt_file); + return 1; + } + args.prompt = prompt_file_contents.c_str(); + } + if (args.backend != nullptr && std::strlen(args.backend) > 0) { setenv("ED_BACKEND", args.backend, 1); } diff --git a/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp b/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp index a00b379..8ffae7d 100644 --- a/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp +++ b/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp @@ -610,15 +610,22 @@ struct AudioVAE : public GGMLBlock { } ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) { GGML_ASSERT(waveform->ne[1] == 2); - waveform = ggml_reshape_3d(ctx->ggml_ctx, waveform, waveform->ne[0], 1, waveform->ne[1]); - auto x = std::dynamic_pointer_cast(blocks["encoder"])->forward(ctx, waveform); - x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); - x = std::dynamic_pointer_cast(blocks["pre_block"])->forward(ctx, x); - x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); - auto z = std::dynamic_pointer_cast(blocks["mean_proj"])->forward(ctx, x); auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1); auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1); - return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std), 0, 2, 1, 3)); + ggml_tensor* encoded = nullptr; + for (int64_t channel = 0; channel < waveform->ne[1]; ++channel) { + auto x = ggml_ext_slice(ctx->ggml_ctx, waveform, 1, channel, channel + 1); + x = ggml_reshape_3d(ctx->ggml_ctx, x, x->ne[0], 1, 1); + x = std::dynamic_pointer_cast(blocks["encoder"])->forward(ctx, x); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); + x = std::dynamic_pointer_cast(blocks["pre_block"])->forward(ctx, x); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); + x = std::dynamic_pointer_cast(blocks["mean_proj"])->forward(ctx, x); + x = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, x, mean), std); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); + encoded = encoded == nullptr ? x : ggml_concat(ctx->ggml_ctx, encoded, x, 1); + } + return encoded; } ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) { GGML_ASSERT(latent->ne[1] == 2 && latent->ne[2] == kLatentChannels); diff --git a/src/dit_models/components/text_encoders/llm.hpp b/src/dit_models/components/text_encoders/llm.hpp index 992d7b8..f50f5bc 100644 --- a/src/dit_models/components/text_encoders/llm.hpp +++ b/src/dit_models/components/text_encoders/llm.hpp @@ -1101,7 +1101,11 @@ namespace LLM { if (deepstack_it != deepstack_visual_indexes.end()) { size_t deepstack_index = static_cast(std::distance(deepstack_visual_indexes.begin(), deepstack_it)); auto deepstack_merger = std::dynamic_pointer_cast(blocks["deepstack_merger_list." + std::to_string(deepstack_index)]); - deepstack_outputs.push_back(deepstack_merger->forward(ctx, x)); + auto deepstack_output = deepstack_merger->forward(ctx, x); + if (debug_target == "deepstack" + std::to_string(deepstack_index)) { + return {deepstack_output}; + } + deepstack_outputs.push_back(deepstack_output); } sd::ggml_graph_cut::mark_graph_cut(x, "llm.vision.blocks." + std::to_string(i), "x"); if (debug_target == "block" + std::to_string(i)) { @@ -1534,6 +1538,9 @@ namespace LLM { input_embed = ggml_cast(ctx->ggml_ctx, input_embed, embed_type); } x = input_embed; + if (debug_target == "after_image_embed") { + return x; + } } if (out_layers.find(0) != out_layers.end()) { @@ -1642,9 +1649,10 @@ namespace LLM { ggml_tensor* window_index, ggml_tensor* window_inverse_index, ggml_tensor* window_mask, - ggml_tensor* pos_embeds = nullptr) { + ggml_tensor* pos_embeds = nullptr, + const std::string& debug_target = "") { GGML_ASSERT(enable_vision); - return vision_model()->forward_outputs(ctx, pixel_values, pe, window_index, window_inverse_index, window_mask, nullptr, pos_embeds); + return vision_model()->forward_outputs(ctx, pixel_values, pe, window_index, window_inverse_index, window_mask, nullptr, pos_embeds, debug_target); } }; @@ -2270,7 +2278,8 @@ namespace LLM { nullptr, nullptr, nullptr, - pos_embeds); + pos_embeds, + debug_target); ggml_build_forward_expand(gf, outputs[0]); return gf; } @@ -2506,7 +2515,47 @@ namespace LLM { } const int64_t image_width = image.shape()[0]; const int64_t image_height = image.shape()[1]; + qwen_align_log_tensor_stats("vision.normalized_image", image); const auto pixel_values = process_image_patches_host(image); + qwen_align_log_tensor_stats("vision.pixel_values", pixel_values); + if (qwen_align_debug_enabled()) { + const char* dump_targets = std::getenv("ED_QWEN_ALIGN_DUMP_TARGETS"); + const std::vector debug_targets = { + "patch_embed", + "block0", + "block0.input", + "block0.norm1", + "block0.attn.qkv", + "block0.attn.q_rope", + "block0.attn.out", + "block0.after_attn", + "block0.norm2", + "block0.mlp.out", + "block0.after_mlp", + "block8", + "block16", + "block24", + "block26", + "deepstack0", + "deepstack1", + "deepstack2", + "merged", + "merger.input", + "merger.out", + }; + for (const std::string& target : debug_targets) { + const std::string full_name = "vision." + target; + if (!qwen_align_csv_contains(dump_targets, full_name.c_str()) && + !qwen_align_csv_contains(dump_targets, target.c_str())) { + continue; + } + auto get_debug_graph = [&]() -> ggml_cgraph* { + return build_encode_image_graph(pixel_values, image_width, image_height, target); + }; + auto debug_tensor = take_or_empty(GGMLRunner::compute(get_debug_graph, n_threads, false)); + qwen_align_log_tensor_stats(full_name.c_str(), debug_tensor); + } + } auto get_graph = [&]() -> ggml_cgraph* { ggml_cgraph* graph = new_graph_custom(LLM_GRAPH_SIZE); auto pixels = make_input(pixel_values); diff --git a/src/dit_models/models/minimax_h3_full.hpp b/src/dit_models/models/minimax_h3_full.hpp index 276d7a5..251fff7 100644 --- a/src/dit_models/models/minimax_h3_full.hpp +++ b/src/dit_models/models/minimax_h3_full.hpp @@ -829,6 +829,10 @@ namespace MiniMaxH3 { ggml_tensor* audio_slope, const std::string& debug_target = {}, ggml_tensor** debug_output = nullptr) { + if (debug_output != nullptr && debug_target == "position_ids") { + *debug_output = ggml_ext_scale(ctx->ggml_ctx, position_ids, 1.0f); + return {nullptr, nullptr}; + } auto video_proj = std::dynamic_pointer_cast(blocks["video_patch_proj"]); auto audio_proj = std::dynamic_pointer_cast(blocks["audio_patch_proj"]); @@ -880,6 +884,11 @@ namespace MiniMaxH3 { std::pair target_audio_range = {audio_offset, audio_offset + audio->ne[1]}; audio_rows = audio_rows == nullptr ? audio : ggml_concat(ctx->ggml_ctx, audio_rows, audio, 1); auto audio_embeds = audio_proj->forward(ctx, audio_rows); + if (debug_output != nullptr && debug_target == "audio_rows") { + *debug_output = audio_rows; + } else if (debug_output != nullptr && debug_target == "audio_embeds") { + *debug_output = audio_embeds; + } context = refine_context(ctx, context); ggml_tensor* h = nullptr; diff --git a/src/dit_models/pipelines/minimax_h3_pipeline.cpp b/src/dit_models/pipelines/minimax_h3_pipeline.cpp index 6ddcc3d..f21ef30 100644 --- a/src/dit_models/pipelines/minimax_h3_pipeline.cpp +++ b/src/dit_models/pipelines/minimax_h3_pipeline.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,61 @@ namespace edgedit { namespace { constexpr int H3_REF_IMAGE_SHORT_EDGE = 2048; +constexpr int64_t H3_MIN_GENERATION_PIXELS = 256 * 256; + +std::vector h3_resample_audio_sinc(const ed_audio_t& source, + uint64_t source_samples, + uint32_t source_channel, + uint32_t target_sample_rate) { + if (source.sample_rate == target_sample_rate) { + std::vector output(source_samples); + for (uint64_t sample = 0; sample < source_samples; ++sample) { + output[sample] = std::clamp(source.data[sample * source.channels + source_channel], -1.f, 1.f); + } + return output; + } + + const uint32_t divisor = std::gcd(source.sample_rate, target_sample_rate); + const int64_t original_rate = source.sample_rate / divisor; + const int64_t target_rate = target_sample_rate / divisor; + constexpr double filter_width = 6.0; + constexpr double rolloff = 0.99; + const double base_rate = std::min(original_rate, target_rate) * rolloff; + const int64_t width = static_cast(std::ceil(filter_width * original_rate / base_rate)); + const int64_t kernel_size = width * 2 + original_rate; + const uint64_t target_samples = (source_samples * static_cast(target_rate) + original_rate - 1) / + static_cast(original_rate); + std::vector kernels(static_cast(target_rate * kernel_size)); + for (int64_t phase = 0; phase < target_rate; ++phase) { + for (int64_t tap = 0; tap < kernel_size; ++tap) { + double time = (static_cast(tap - width) / original_rate - + static_cast(phase) / target_rate) * base_rate; + time = std::clamp(time, -filter_width, filter_width); + const double window = std::pow(std::cos(time * M_PI / filter_width / 2.0), 2.0); + const double angle = time * M_PI; + const double sinc = std::abs(angle) < 1e-12 ? 1.0 : std::sin(angle) / angle; + kernels[static_cast(phase * kernel_size + tap)] = + static_cast(sinc * window * base_rate / original_rate); + } + } + + std::vector output(target_samples, 0.f); + for (uint64_t sample = 0; sample < target_samples; ++sample) { + const int64_t frame = static_cast(sample / target_rate); + const int64_t phase = static_cast(sample % target_rate); + float value = 0.f; + for (int64_t tap = 0; tap < kernel_size; ++tap) { + const int64_t input_index = frame * original_rate + tap - width; + if (input_index < 0 || input_index >= static_cast(source_samples)) { + continue; + } + value += source.data[static_cast(input_index) * source.channels + source_channel] * + kernels[static_cast(phase * kernel_size + tap)]; + } + output[sample] = std::clamp(value, -1.f, 1.f); + } + return output; +} struct ScopedEnvVar { std::string name; @@ -319,6 +375,29 @@ void h3_dump_debug_tensor_if_requested(const std::string& name, const sd::Tensor } } +bool h3_load_debug_tensor_if_requested(const std::string& name, sd::Tensor* tensor) { + const char* base_path = std::getenv("ED_MINIMAX_H3_DEBUG_LOAD"); + if (base_path == nullptr || base_path[0] == '\0' || tensor == nullptr || tensor->empty()) { + return false; + } + std::ifstream input(std::string(base_path) + "." + name + ".f32.bin", std::ios::binary | std::ios::ate); + if (!input) { + return false; + } + const std::streamsize expected = static_cast(tensor->numel() * sizeof(float)); + if (input.tellg() != expected) { + LOG_ERROR("MiniMax-H3 debug tensor '%s' has %lld bytes, expected %lld", + name.c_str(), + static_cast(input.tellg()), + static_cast(expected)); + return false; + } + input.seekg(0); + input.read(reinterpret_cast(tensor->data()), expected); + LOG_INFO("MiniMax-H3 loaded debug tensor '%s'", name.c_str()); + return input.good(); +} + sd::Tensor h3_sample_vae_moments(const sd::Tensor& moments, std::shared_ptr rng) { if (moments.empty() || moments.shape().size() < 4 || moments.shape()[3] != 48) { @@ -379,10 +458,13 @@ sd::Tensor h3_encode_vae_condition(MiniMaxH3VAE::MiniMaxH3VideoVAERunner* return encoded; } -void h3_apply_condition_noise(sd::Tensor* latent, uint64_t seed) { - auto rng = std::make_shared(seed); +void h3_apply_condition_noise(sd::Tensor* latent, + const std::shared_ptr& rng, + const std::string& debug_name) { + auto noise = sd::randn_like(*latent, rng); + h3_load_debug_tensor_if_requested(debug_name, &noise); *latent = *latent * MiniMaxH3::VISUAL_COND_TIMESTEP + - sd::randn_like(*latent, rng) * (1.0f - MiniMaxH3::VISUAL_COND_TIMESTEP); + noise * (1.0f - MiniMaxH3::VISUAL_COND_TIMESTEP); } sd::Tensor h3_pack_audio_and_video_latents(const sd::Tensor& video, @@ -813,12 +895,6 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, if (vision_start_tokens.size() != 1 || vision_end_tokens.size() != 1 || image_pad_tokens.size() != 1) { return set_minimax_error(error, "MiniMax-H3 tokenizer special tokens are invalid"); } - auto append_text = [&](const std::string& text) { - pending_text += text; - if (verify_token_build) { - legacy_presentation += text; - } - }; auto flush_text = [&]() { if (pending_text.empty()) { return; @@ -831,6 +907,13 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, tokens.insert(tokens.end(), text_tokens.begin(), text_tokens.end()); pending_text.clear(); }; + auto append_text = [&](const std::string& text) { + flush_text(); + pending_text = text; + if (verify_token_build) { + legacy_presentation += text; + } + }; auto append_vision = [&](int64_t count) { flush_text(); tokens.push_back(vision_start_tokens[0]); @@ -985,23 +1068,19 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, flush_text(); if (profile != nullptr) profile->text_tokenize_ms += tokenization_ms; if (verify_token_build) { - const auto legacy_tokens = conditioner_->tokenizer->tokenize(legacy_presentation, nullptr, true, 0, 0, false); - if (legacy_tokens != tokens) { - size_t mismatch = 0; - while (mismatch < legacy_tokens.size() && mismatch < tokens.size() && - legacy_tokens[mismatch] == tokens[mismatch]) { - ++mismatch; - } - std::ostringstream message; - message << "MiniMax-H3 direct token build mismatch at " << mismatch - << ": direct=" << tokens.size() << " legacy=" << legacy_tokens.size(); - return set_minimax_error(error, message.str().c_str()); - } - LOG_INFO("MiniMax-H3 direct token build verified: tokens=%zu", tokens.size()); + LOG_INFO("MiniMax-H3 segmented presentation token build: tokens=%zu bytes=%zu", + tokens.size(), + legacy_presentation.size()); } if (tokens.empty()) { return set_minimax_error(error, "MiniMax-H3 prompt tokenization produced no tokens"); } + if (const char* token_dump = std::getenv("ED_MINIMAX_H3_DUMP_TOKEN_IDS"); + token_dump != nullptr && token_dump[0] != '\0') { + std::ofstream output(token_dump, std::ios::binary); + output.write(reinterpret_cast(tokens.data()), + static_cast(tokens.size() * sizeof(tokens[0]))); + } std::vector tags(tokens.size(), 1); for (const auto& [index, image_embed] : image_embeds) { const int64_t begin = std::max(0, index - 1); @@ -1041,6 +1120,7 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, const int64_t tag_count = static_cast(tags.size()); *token_tags = sd::Tensor({tag_count}, std::move(tags)); h3_trace_tensor("text_context", *context); + h3_dump_debug_tensor_if_requested("text_context", *context); return true; } @@ -1209,6 +1289,12 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t set_minimax_error(error, "MiniMax-H3 width and height must be positive multiples of 32"); return ED_STATUS_INVALID_ARGUMENT; } + if (static_cast(params->width) * params->height < H3_MIN_GENERATION_PIXELS) { + set_minimax_error(error, + "MiniMax-H3 output canvas must contain at least 65536 pixels (for example 256x256); " + "the official recommended output uses a 768-pixel short edge"); + return ED_STATUS_INVALID_ARGUMENT; + } const int frames = std::max(5, params->frames); if (frames % 17 != 5) { set_minimax_error(error, "MiniMax-H3 frame count must satisfy 17k + 5"); @@ -1321,6 +1407,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t std::vector> keyframe_latents; std::vector keyframe_indices; auto request_rng = std::make_shared(static_cast(resolved_seed)); + int condition_noise_index = 0; if (stage_video_vae_lifecycle_ && need_video_vae_for_conditioning && !vae_->stage_params_for_phase()) { set_minimax_error(error, "MiniMax-H3 failed to stage video VAE for conditioning"); return ED_STATUS_OUT_OF_MEMORY; @@ -1351,7 +1438,9 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t return set_minimax_error(error, "MiniMax-H3 keyframe VAE encode failed"); } sd::Tensor latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, static_cast(resolved_seed)); + h3_apply_condition_noise(&latent, + request_rng, + "condition_noise_" + std::to_string(condition_noise_index++)); h3_trace_tensor((std::string(name) + "_keyframe_latent").c_str(), latent); keyframe_latents.push_back(std::move(latent)); keyframe_indices.push_back(frame_index); @@ -1368,18 +1457,16 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t auto encode_reference_audio = [&](const ed_audio_t& source, int32_t* index) -> bool { if (audio_vae_ == nullptr || source.data == nullptr || source.sample_count == 0 || source.channels == 0 || source.sample_rate == 0) return false; const int64_t prepare_begin = profile_ptr != nullptr ? ggml_time_ms() : 0; - const uint64_t samples = std::max(1, (source.sample_count * 32000ULL + source.sample_rate / 2) / source.sample_rate); + const uint64_t max_source_samples = static_cast( + static_cast(frames) * source.sample_rate / 24.0L); + const uint64_t source_samples = std::max(1, std::min(source.sample_count, max_source_samples)); + const uint64_t samples = std::max(1, (source_samples * 32000ULL + source.sample_rate - 1) / source.sample_rate); sd::Tensor waveform({static_cast(((samples + 799) / 800) * 800), 2, 1, 1}); - for (uint64_t sample = 0; sample < samples; ++sample) { - const double position = static_cast(sample) * source.sample_rate / 32000.0; - const uint64_t first = std::min(static_cast(position), source.sample_count - 1); - const uint64_t second = std::min(first + 1, source.sample_count - 1); - const float fraction = static_cast(position - first); - for (uint32_t channel = 0; channel < 2; ++channel) { - const uint32_t source_channel = source.channels == 1 ? 0 : std::min(channel, source.channels - 1); - const float a = source.data[first * source.channels + source_channel]; - const float b = source.data[second * source.channels + source_channel]; - waveform.index(sample, channel, 0, 0) = std::clamp(a + (b - a) * fraction, -1.f, 1.f); + for (uint32_t channel = 0; channel < 2; ++channel) { + const uint32_t source_channel = source.channels == 1 ? 0 : std::min(channel, source.channels - 1); + const auto resampled = h3_resample_audio_sinc(source, source_samples, source_channel, 32000); + for (uint64_t sample = 0; sample < std::min(samples, resampled.size()); ++sample) { + waveform.index(sample, channel, 0, 0) = resampled[sample]; } } if (profile_ptr != nullptr) profile.reference_audio_prepare_ms += ggml_time_ms() - prepare_begin; @@ -1387,6 +1474,8 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t auto encoded = audio_vae_->encode(runtime_->n_threads(), waveform); if (profile_ptr != nullptr) profile.reference_audio_vae_encode_ms += ggml_time_ms() - vae_encode_begin; if (encoded.empty()) return false; + h3_trace_tensor("reference_audio_latent", encoded); + h3_dump_debug_tensor_if_requested("reference_audio_latent", encoded); *index = static_cast(reference_audio_latents.size()); reference_audio_latents.push_back(std::move(encoded)); return true; @@ -1422,7 +1511,9 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t return ED_STATUS_GENERATION_FAILED; } sd::Tensor latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, static_cast(resolved_seed)); + h3_apply_condition_noise(&latent, + request_rng, + "condition_noise_" + std::to_string(condition_noise_index++)); h3_trace_tensor(("ref_image_" + std::to_string(reference_index) + "_latent").c_str(), latent); const int32_t encoded_image_index = static_cast(reference_latents.size()); reference_latents.push_back(std::move(latent)); @@ -1475,7 +1566,9 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t if (profile_ptr != nullptr) profile.reference_video_vae_encode_ms += ggml_time_ms() - vae_encode_begin; if (vae_latent.empty()) { set_minimax_error(error, "MiniMax-H3 Ref2VA video VAE encode failed"); return ED_STATUS_GENERATION_FAILED; } auto latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, static_cast(resolved_seed)); + h3_apply_condition_noise(&latent, + request_rng, + "condition_noise_" + std::to_string(condition_noise_index++)); h3_trace_tensor(("ref_video_" + std::to_string(video_index) + "_latent").c_str(), latent); const int32_t encoded_video_index = static_cast(reference_latents.size()); reference_latents.push_back(std::move(latent)); @@ -1535,9 +1628,12 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t const int64_t noise_begin = profile_ptr != nullptr ? ggml_time_ms() : 0; video = sd::randn_like(video, request_rng); audio = sd::randn_like(audio, request_rng); + h3_load_debug_tensor_if_requested("target_video_noise", &video); + h3_load_debug_tensor_if_requested("target_audio_noise", &audio); sd::Tensor packed = h3_pack_audio_and_video_latents(video, audio); if (profile_ptr != nullptr) profile.noise_init_ms = ggml_time_ms() - noise_begin; h3_trace_tensor("initial_packed_noise", packed); + h3_dump_debug_tensor_if_requested("initial_packed_noise", packed); if (h3_trace_enabled()) { auto reference_kind_name = [](MiniMaxH3ReferenceKind kind) { switch (kind) { @@ -1607,12 +1703,11 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t const float sigma = sigma_at(step); const float sigma_next = sigma_at(step + 1); sd::Tensor model_packed = packed; - float audio_sigma = sigma; - float audio_slope = 1.0f; + const float audio_sigma = MiniMaxH3::time_shift_sigma(sigma, video_sigma_shift, 3.0f); + const float audio_sigma_next = MiniMaxH3::time_shift_sigma(sigma_next, video_sigma_shift, 3.0f); + const float audio_slope = MiniMaxH3::time_shift_slope(sigma, video_sigma_shift, 3.0f); float audio_scale = 1.0f; if (sampler == ED_SAMPLER_RES_MULTISTEP) { - audio_sigma = MiniMaxH3::time_shift_sigma(sigma, video_sigma_shift, 3.0f); - audio_slope = MiniMaxH3::time_shift_slope(sigma, video_sigma_shift, 3.0f); audio_scale = video_sigma_shift / 3.0f; auto model_av = diffusion_->split_av_latents(packed, audio_length); model_av.second *= audio_sigma / sigma; @@ -1706,7 +1801,11 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t old_sigma_down = sigma_next; have_old_denoised = true; } else { - packed += velocity * (sigma_next - sigma); + auto packed_av = diffusion_->split_av_latents(packed, audio_length); + auto velocity_av = diffusion_->split_av_latents(velocity, audio_length); + packed_av.first += velocity_av.first * (sigma_next - sigma); + packed_av.second += (velocity_av.second / audio_slope) * (audio_sigma_next - audio_sigma); + packed = h3_pack_audio_and_video_latents(packed_av.first, packed_av.second); } if (h3_trace_enabled()) { h3_trace_tensor(("step_" + std::to_string(step) + "_packed").c_str(), packed); @@ -1717,6 +1816,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t if (sampler == ED_SAMPLER_RES_MULTISTEP) { av.second /= video_sigma_shift / 3.0f; } + h3_dump_debug_tensor_if_requested("final_audio_latent", av.second); ed_status_t status = decode_video_latent(av.first, frames, out, profile_ptr, error); if (status != ED_STATUS_OK) { return status; From 246a0c164ec4535839dd9622a34834074599895d Mon Sep 17 00:00:00 2001 From: Komorebi623 Date: Tue, 18 Aug 2026 19:57:53 +0800 Subject: [PATCH 2/3] chore(minimax-h3): remove temporary alignment debug hooks Remove temporary tensor-load/dump and Qwen debug-target plumbing that was accidentally included with the Ref2VA fixes. Keep the functional audio, conditioning, prompt-file, and resolution changes intact. --- .../components/text_encoders/llm.hpp | 57 ++----------------- src/dit_models/models/minimax_h3_full.hpp | 9 --- .../pipelines/minimax_h3_pipeline.cpp | 53 ++--------------- 3 files changed, 8 insertions(+), 111 deletions(-) diff --git a/src/dit_models/components/text_encoders/llm.hpp b/src/dit_models/components/text_encoders/llm.hpp index f50f5bc..992d7b8 100644 --- a/src/dit_models/components/text_encoders/llm.hpp +++ b/src/dit_models/components/text_encoders/llm.hpp @@ -1101,11 +1101,7 @@ namespace LLM { if (deepstack_it != deepstack_visual_indexes.end()) { size_t deepstack_index = static_cast(std::distance(deepstack_visual_indexes.begin(), deepstack_it)); auto deepstack_merger = std::dynamic_pointer_cast(blocks["deepstack_merger_list." + std::to_string(deepstack_index)]); - auto deepstack_output = deepstack_merger->forward(ctx, x); - if (debug_target == "deepstack" + std::to_string(deepstack_index)) { - return {deepstack_output}; - } - deepstack_outputs.push_back(deepstack_output); + deepstack_outputs.push_back(deepstack_merger->forward(ctx, x)); } sd::ggml_graph_cut::mark_graph_cut(x, "llm.vision.blocks." + std::to_string(i), "x"); if (debug_target == "block" + std::to_string(i)) { @@ -1538,9 +1534,6 @@ namespace LLM { input_embed = ggml_cast(ctx->ggml_ctx, input_embed, embed_type); } x = input_embed; - if (debug_target == "after_image_embed") { - return x; - } } if (out_layers.find(0) != out_layers.end()) { @@ -1649,10 +1642,9 @@ namespace LLM { ggml_tensor* window_index, ggml_tensor* window_inverse_index, ggml_tensor* window_mask, - ggml_tensor* pos_embeds = nullptr, - const std::string& debug_target = "") { + ggml_tensor* pos_embeds = nullptr) { GGML_ASSERT(enable_vision); - return vision_model()->forward_outputs(ctx, pixel_values, pe, window_index, window_inverse_index, window_mask, nullptr, pos_embeds, debug_target); + return vision_model()->forward_outputs(ctx, pixel_values, pe, window_index, window_inverse_index, window_mask, nullptr, pos_embeds); } }; @@ -2278,8 +2270,7 @@ namespace LLM { nullptr, nullptr, nullptr, - pos_embeds, - debug_target); + pos_embeds); ggml_build_forward_expand(gf, outputs[0]); return gf; } @@ -2515,47 +2506,7 @@ namespace LLM { } const int64_t image_width = image.shape()[0]; const int64_t image_height = image.shape()[1]; - qwen_align_log_tensor_stats("vision.normalized_image", image); const auto pixel_values = process_image_patches_host(image); - qwen_align_log_tensor_stats("vision.pixel_values", pixel_values); - if (qwen_align_debug_enabled()) { - const char* dump_targets = std::getenv("ED_QWEN_ALIGN_DUMP_TARGETS"); - const std::vector debug_targets = { - "patch_embed", - "block0", - "block0.input", - "block0.norm1", - "block0.attn.qkv", - "block0.attn.q_rope", - "block0.attn.out", - "block0.after_attn", - "block0.norm2", - "block0.mlp.out", - "block0.after_mlp", - "block8", - "block16", - "block24", - "block26", - "deepstack0", - "deepstack1", - "deepstack2", - "merged", - "merger.input", - "merger.out", - }; - for (const std::string& target : debug_targets) { - const std::string full_name = "vision." + target; - if (!qwen_align_csv_contains(dump_targets, full_name.c_str()) && - !qwen_align_csv_contains(dump_targets, target.c_str())) { - continue; - } - auto get_debug_graph = [&]() -> ggml_cgraph* { - return build_encode_image_graph(pixel_values, image_width, image_height, target); - }; - auto debug_tensor = take_or_empty(GGMLRunner::compute(get_debug_graph, n_threads, false)); - qwen_align_log_tensor_stats(full_name.c_str(), debug_tensor); - } - } auto get_graph = [&]() -> ggml_cgraph* { ggml_cgraph* graph = new_graph_custom(LLM_GRAPH_SIZE); auto pixels = make_input(pixel_values); diff --git a/src/dit_models/models/minimax_h3_full.hpp b/src/dit_models/models/minimax_h3_full.hpp index 251fff7..276d7a5 100644 --- a/src/dit_models/models/minimax_h3_full.hpp +++ b/src/dit_models/models/minimax_h3_full.hpp @@ -829,10 +829,6 @@ namespace MiniMaxH3 { ggml_tensor* audio_slope, const std::string& debug_target = {}, ggml_tensor** debug_output = nullptr) { - if (debug_output != nullptr && debug_target == "position_ids") { - *debug_output = ggml_ext_scale(ctx->ggml_ctx, position_ids, 1.0f); - return {nullptr, nullptr}; - } auto video_proj = std::dynamic_pointer_cast(blocks["video_patch_proj"]); auto audio_proj = std::dynamic_pointer_cast(blocks["audio_patch_proj"]); @@ -884,11 +880,6 @@ namespace MiniMaxH3 { std::pair target_audio_range = {audio_offset, audio_offset + audio->ne[1]}; audio_rows = audio_rows == nullptr ? audio : ggml_concat(ctx->ggml_ctx, audio_rows, audio, 1); auto audio_embeds = audio_proj->forward(ctx, audio_rows); - if (debug_output != nullptr && debug_target == "audio_rows") { - *debug_output = audio_rows; - } else if (debug_output != nullptr && debug_target == "audio_embeds") { - *debug_output = audio_embeds; - } context = refine_context(ctx, context); ggml_tensor* h = nullptr; diff --git a/src/dit_models/pipelines/minimax_h3_pipeline.cpp b/src/dit_models/pipelines/minimax_h3_pipeline.cpp index f21ef30..e6e2aea 100644 --- a/src/dit_models/pipelines/minimax_h3_pipeline.cpp +++ b/src/dit_models/pipelines/minimax_h3_pipeline.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -375,29 +374,6 @@ void h3_dump_debug_tensor_if_requested(const std::string& name, const sd::Tensor } } -bool h3_load_debug_tensor_if_requested(const std::string& name, sd::Tensor* tensor) { - const char* base_path = std::getenv("ED_MINIMAX_H3_DEBUG_LOAD"); - if (base_path == nullptr || base_path[0] == '\0' || tensor == nullptr || tensor->empty()) { - return false; - } - std::ifstream input(std::string(base_path) + "." + name + ".f32.bin", std::ios::binary | std::ios::ate); - if (!input) { - return false; - } - const std::streamsize expected = static_cast(tensor->numel() * sizeof(float)); - if (input.tellg() != expected) { - LOG_ERROR("MiniMax-H3 debug tensor '%s' has %lld bytes, expected %lld", - name.c_str(), - static_cast(input.tellg()), - static_cast(expected)); - return false; - } - input.seekg(0); - input.read(reinterpret_cast(tensor->data()), expected); - LOG_INFO("MiniMax-H3 loaded debug tensor '%s'", name.c_str()); - return input.good(); -} - sd::Tensor h3_sample_vae_moments(const sd::Tensor& moments, std::shared_ptr rng) { if (moments.empty() || moments.shape().size() < 4 || moments.shape()[3] != 48) { @@ -459,10 +435,8 @@ sd::Tensor h3_encode_vae_condition(MiniMaxH3VAE::MiniMaxH3VideoVAERunner* } void h3_apply_condition_noise(sd::Tensor* latent, - const std::shared_ptr& rng, - const std::string& debug_name) { + const std::shared_ptr& rng) { auto noise = sd::randn_like(*latent, rng); - h3_load_debug_tensor_if_requested(debug_name, &noise); *latent = *latent * MiniMaxH3::VISUAL_COND_TIMESTEP + noise * (1.0f - MiniMaxH3::VISUAL_COND_TIMESTEP); } @@ -1075,12 +1049,6 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, if (tokens.empty()) { return set_minimax_error(error, "MiniMax-H3 prompt tokenization produced no tokens"); } - if (const char* token_dump = std::getenv("ED_MINIMAX_H3_DUMP_TOKEN_IDS"); - token_dump != nullptr && token_dump[0] != '\0') { - std::ofstream output(token_dump, std::ios::binary); - output.write(reinterpret_cast(tokens.data()), - static_cast(tokens.size() * sizeof(tokens[0]))); - } std::vector tags(tokens.size(), 1); for (const auto& [index, image_embed] : image_embeds) { const int64_t begin = std::max(0, index - 1); @@ -1120,7 +1088,6 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, const int64_t tag_count = static_cast(tags.size()); *token_tags = sd::Tensor({tag_count}, std::move(tags)); h3_trace_tensor("text_context", *context); - h3_dump_debug_tensor_if_requested("text_context", *context); return true; } @@ -1407,7 +1374,6 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t std::vector> keyframe_latents; std::vector keyframe_indices; auto request_rng = std::make_shared(static_cast(resolved_seed)); - int condition_noise_index = 0; if (stage_video_vae_lifecycle_ && need_video_vae_for_conditioning && !vae_->stage_params_for_phase()) { set_minimax_error(error, "MiniMax-H3 failed to stage video VAE for conditioning"); return ED_STATUS_OUT_OF_MEMORY; @@ -1438,9 +1404,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t return set_minimax_error(error, "MiniMax-H3 keyframe VAE encode failed"); } sd::Tensor latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, - request_rng, - "condition_noise_" + std::to_string(condition_noise_index++)); + h3_apply_condition_noise(&latent, request_rng); h3_trace_tensor((std::string(name) + "_keyframe_latent").c_str(), latent); keyframe_latents.push_back(std::move(latent)); keyframe_indices.push_back(frame_index); @@ -1475,7 +1439,6 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t if (profile_ptr != nullptr) profile.reference_audio_vae_encode_ms += ggml_time_ms() - vae_encode_begin; if (encoded.empty()) return false; h3_trace_tensor("reference_audio_latent", encoded); - h3_dump_debug_tensor_if_requested("reference_audio_latent", encoded); *index = static_cast(reference_audio_latents.size()); reference_audio_latents.push_back(std::move(encoded)); return true; @@ -1511,9 +1474,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t return ED_STATUS_GENERATION_FAILED; } sd::Tensor latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, - request_rng, - "condition_noise_" + std::to_string(condition_noise_index++)); + h3_apply_condition_noise(&latent, request_rng); h3_trace_tensor(("ref_image_" + std::to_string(reference_index) + "_latent").c_str(), latent); const int32_t encoded_image_index = static_cast(reference_latents.size()); reference_latents.push_back(std::move(latent)); @@ -1566,9 +1527,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t if (profile_ptr != nullptr) profile.reference_video_vae_encode_ms += ggml_time_ms() - vae_encode_begin; if (vae_latent.empty()) { set_minimax_error(error, "MiniMax-H3 Ref2VA video VAE encode failed"); return ED_STATUS_GENERATION_FAILED; } auto latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, - request_rng, - "condition_noise_" + std::to_string(condition_noise_index++)); + h3_apply_condition_noise(&latent, request_rng); h3_trace_tensor(("ref_video_" + std::to_string(video_index) + "_latent").c_str(), latent); const int32_t encoded_video_index = static_cast(reference_latents.size()); reference_latents.push_back(std::move(latent)); @@ -1628,12 +1587,9 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t const int64_t noise_begin = profile_ptr != nullptr ? ggml_time_ms() : 0; video = sd::randn_like(video, request_rng); audio = sd::randn_like(audio, request_rng); - h3_load_debug_tensor_if_requested("target_video_noise", &video); - h3_load_debug_tensor_if_requested("target_audio_noise", &audio); sd::Tensor packed = h3_pack_audio_and_video_latents(video, audio); if (profile_ptr != nullptr) profile.noise_init_ms = ggml_time_ms() - noise_begin; h3_trace_tensor("initial_packed_noise", packed); - h3_dump_debug_tensor_if_requested("initial_packed_noise", packed); if (h3_trace_enabled()) { auto reference_kind_name = [](MiniMaxH3ReferenceKind kind) { switch (kind) { @@ -1816,7 +1772,6 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t if (sampler == ED_SAMPLER_RES_MULTISTEP) { av.second /= video_sigma_shift / 3.0f; } - h3_dump_debug_tensor_if_requested("final_audio_latent", av.second); ed_status_t status = decode_video_latent(av.first, frames, out, profile_ptr, error); if (status != ED_STATUS_OK) { return status; From 9c0b9f86d4b733bb3bab33971330db40c5204d44 Mon Sep 17 00:00:00 2001 From: Komorebi623 Date: Tue, 18 Aug 2026 20:05:22 +0800 Subject: [PATCH 3/3] fix(minimax-h3): apply audio VAE zero key bias --- .../components/autoencoders/minimax_h3_audio_vae.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp b/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp index 8ffae7d..0a8ecf1 100644 --- a/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp +++ b/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp @@ -531,16 +531,17 @@ struct AudioCausalAttention : public GGMLBlock { static constexpr int64_t in_channels = 2048, out_channels = 32, num_head = 8, head_dim = in_channels / num_head; AudioCausalAttention() { blocks["qkv"] = std::make_shared(in_channels, in_channels * 3, false); blocks["proj"] = std::make_shared(out_channels, out_channels, true); } void init_params(ggml_context* ctx, const String2TensorStorage& storage = {}, const std::string prefix = "") override { - GGMLBlock::init_params(ctx, storage, prefix); params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); + GGMLBlock::init_params(ctx, storage, prefix); params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["zero_k_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); } ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { auto qkv = ggml_ext_chunk(ctx->ggml_ctx, std::dynamic_pointer_cast(blocks["qkv"])->forward(ctx, x), 3, 0); auto shape_bias = [&](ggml_tensor* value) { return ggml_reshape_4d(ctx->ggml_ctx, value, value->ne[0], 1, 1, 1); }; auto q = ggml_add(ctx->ggml_ctx, qkv[0], shape_bias(params["q_bias"])); + auto k = ggml_add(ctx->ggml_ctx, qkv[1], shape_bias(params["zero_k_bias"])); auto v = ggml_add(ctx->ggml_ctx, qkv[2], shape_bias(params["v_bias"])); const int64_t sequence = x->ne[1]; auto mask = ggml_diag_mask_inf(ctx->ggml_ctx, ggml_ext_zeros(ctx->ggml_ctx, sequence, sequence, 1, 1), 0); - auto out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, qkv[1], v, num_head, mask, false, ctx->flash_attn_enabled); + auto out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_head, mask, false, ctx->flash_attn_enabled); const int64_t batch = out->ne[2] * out->ne[3]; out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, ggml_reshape_4d(ctx->ggml_ctx, out, head_dim, num_head, sequence, batch), 1, 0, 2, 3)); out = ggml_mean(ctx->ggml_ctx, out);