diff --git a/docs/minimax-h3.md b/docs/minimax-h3.md index ab6d883..1890517 100644 --- a/docs/minimax-h3.md +++ b/docs/minimax-h3.md @@ -127,6 +127,16 @@ seconds. The resolved value is printed before generation. Use `--video-frames ` when an exact legal frame count is required. The two options are mutually exclusive. +## Output resolution + +Output width and height must be multiples of 32. MiniMax-H3 is trained and +released for 768p base generation; the official workflow uses a 768-pixel +short edge by default. edge-dit.cpp additionally rejects output canvases below +65,536 pixels (for example, `128x128`) because the model can produce +structurally invalid block mosaics at those extremely small sizes. Use at least +`256x256`; use a 768-pixel short edge for the model's recommended quality +range. + ## Usage Set component paths for the desired precision. The video and audio VAE files are diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 0fc3b69..5b977e7 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -871,6 +872,24 @@ int main(int argc, char** argv) { return launch_status; } + std::string prompt_file_contents; + if ((args.prompt == nullptr || args.prompt[0] == '\0') && + args.prompt_file != nullptr && args.prompt_file[0] != '\0') { + std::ifstream prompt_file(args.prompt_file, std::ios::binary); + if (!prompt_file.is_open()) { + std::fprintf(stderr, "failed to open prompt file: %s\n", args.prompt_file); + return 1; + } + std::ostringstream prompt_stream; + prompt_stream << prompt_file.rdbuf(); + prompt_file_contents = prompt_stream.str(); + if (prompt_file_contents.empty()) { + std::fprintf(stderr, "prompt file is empty: %s\n", args.prompt_file); + return 1; + } + args.prompt = prompt_file_contents.c_str(); + } + if (args.backend != nullptr && std::strlen(args.backend) > 0) { setenv("ED_BACKEND", args.backend, 1); } diff --git a/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp b/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp index a00b379..0a8ecf1 100644 --- a/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp +++ b/src/dit_models/components/autoencoders/minimax_h3_audio_vae.hpp @@ -531,16 +531,17 @@ struct AudioCausalAttention : public GGMLBlock { static constexpr int64_t in_channels = 2048, out_channels = 32, num_head = 8, head_dim = in_channels / num_head; AudioCausalAttention() { blocks["qkv"] = std::make_shared(in_channels, in_channels * 3, false); blocks["proj"] = std::make_shared(out_channels, out_channels, true); } void init_params(ggml_context* ctx, const String2TensorStorage& storage = {}, const std::string prefix = "") override { - GGMLBlock::init_params(ctx, storage, prefix); params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); + GGMLBlock::init_params(ctx, storage, prefix); params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["zero_k_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); } ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { auto qkv = ggml_ext_chunk(ctx->ggml_ctx, std::dynamic_pointer_cast(blocks["qkv"])->forward(ctx, x), 3, 0); auto shape_bias = [&](ggml_tensor* value) { return ggml_reshape_4d(ctx->ggml_ctx, value, value->ne[0], 1, 1, 1); }; auto q = ggml_add(ctx->ggml_ctx, qkv[0], shape_bias(params["q_bias"])); + auto k = ggml_add(ctx->ggml_ctx, qkv[1], shape_bias(params["zero_k_bias"])); auto v = ggml_add(ctx->ggml_ctx, qkv[2], shape_bias(params["v_bias"])); const int64_t sequence = x->ne[1]; auto mask = ggml_diag_mask_inf(ctx->ggml_ctx, ggml_ext_zeros(ctx->ggml_ctx, sequence, sequence, 1, 1), 0); - auto out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, qkv[1], v, num_head, mask, false, ctx->flash_attn_enabled); + auto out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_head, mask, false, ctx->flash_attn_enabled); const int64_t batch = out->ne[2] * out->ne[3]; out = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, ggml_reshape_4d(ctx->ggml_ctx, out, head_dim, num_head, sequence, batch), 1, 0, 2, 3)); out = ggml_mean(ctx->ggml_ctx, out); @@ -610,15 +611,22 @@ struct AudioVAE : public GGMLBlock { } ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) { GGML_ASSERT(waveform->ne[1] == 2); - waveform = ggml_reshape_3d(ctx->ggml_ctx, waveform, waveform->ne[0], 1, waveform->ne[1]); - auto x = std::dynamic_pointer_cast(blocks["encoder"])->forward(ctx, waveform); - x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); - x = std::dynamic_pointer_cast(blocks["pre_block"])->forward(ctx, x); - x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); - auto z = std::dynamic_pointer_cast(blocks["mean_proj"])->forward(ctx, x); auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1); auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1); - return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std), 0, 2, 1, 3)); + ggml_tensor* encoded = nullptr; + for (int64_t channel = 0; channel < waveform->ne[1]; ++channel) { + auto x = ggml_ext_slice(ctx->ggml_ctx, waveform, 1, channel, channel + 1); + x = ggml_reshape_3d(ctx->ggml_ctx, x, x->ne[0], 1, 1); + x = std::dynamic_pointer_cast(blocks["encoder"])->forward(ctx, x); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); + x = std::dynamic_pointer_cast(blocks["pre_block"])->forward(ctx, x); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); + x = std::dynamic_pointer_cast(blocks["mean_proj"])->forward(ctx, x); + x = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, x, mean), std); + x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); + encoded = encoded == nullptr ? x : ggml_concat(ctx->ggml_ctx, encoded, x, 1); + } + return encoded; } ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) { GGML_ASSERT(latent->ne[1] == 2 && latent->ne[2] == kLatentChannels); diff --git a/src/dit_models/pipelines/minimax_h3_pipeline.cpp b/src/dit_models/pipelines/minimax_h3_pipeline.cpp index 6ddcc3d..e6e2aea 100644 --- a/src/dit_models/pipelines/minimax_h3_pipeline.cpp +++ b/src/dit_models/pipelines/minimax_h3_pipeline.cpp @@ -5,10 +5,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include @@ -27,6 +27,61 @@ namespace edgedit { namespace { constexpr int H3_REF_IMAGE_SHORT_EDGE = 2048; +constexpr int64_t H3_MIN_GENERATION_PIXELS = 256 * 256; + +std::vector h3_resample_audio_sinc(const ed_audio_t& source, + uint64_t source_samples, + uint32_t source_channel, + uint32_t target_sample_rate) { + if (source.sample_rate == target_sample_rate) { + std::vector output(source_samples); + for (uint64_t sample = 0; sample < source_samples; ++sample) { + output[sample] = std::clamp(source.data[sample * source.channels + source_channel], -1.f, 1.f); + } + return output; + } + + const uint32_t divisor = std::gcd(source.sample_rate, target_sample_rate); + const int64_t original_rate = source.sample_rate / divisor; + const int64_t target_rate = target_sample_rate / divisor; + constexpr double filter_width = 6.0; + constexpr double rolloff = 0.99; + const double base_rate = std::min(original_rate, target_rate) * rolloff; + const int64_t width = static_cast(std::ceil(filter_width * original_rate / base_rate)); + const int64_t kernel_size = width * 2 + original_rate; + const uint64_t target_samples = (source_samples * static_cast(target_rate) + original_rate - 1) / + static_cast(original_rate); + std::vector kernels(static_cast(target_rate * kernel_size)); + for (int64_t phase = 0; phase < target_rate; ++phase) { + for (int64_t tap = 0; tap < kernel_size; ++tap) { + double time = (static_cast(tap - width) / original_rate - + static_cast(phase) / target_rate) * base_rate; + time = std::clamp(time, -filter_width, filter_width); + const double window = std::pow(std::cos(time * M_PI / filter_width / 2.0), 2.0); + const double angle = time * M_PI; + const double sinc = std::abs(angle) < 1e-12 ? 1.0 : std::sin(angle) / angle; + kernels[static_cast(phase * kernel_size + tap)] = + static_cast(sinc * window * base_rate / original_rate); + } + } + + std::vector output(target_samples, 0.f); + for (uint64_t sample = 0; sample < target_samples; ++sample) { + const int64_t frame = static_cast(sample / target_rate); + const int64_t phase = static_cast(sample % target_rate); + float value = 0.f; + for (int64_t tap = 0; tap < kernel_size; ++tap) { + const int64_t input_index = frame * original_rate + tap - width; + if (input_index < 0 || input_index >= static_cast(source_samples)) { + continue; + } + value += source.data[static_cast(input_index) * source.channels + source_channel] * + kernels[static_cast(phase * kernel_size + tap)]; + } + output[sample] = std::clamp(value, -1.f, 1.f); + } + return output; +} struct ScopedEnvVar { std::string name; @@ -379,10 +434,11 @@ sd::Tensor h3_encode_vae_condition(MiniMaxH3VAE::MiniMaxH3VideoVAERunner* return encoded; } -void h3_apply_condition_noise(sd::Tensor* latent, uint64_t seed) { - auto rng = std::make_shared(seed); +void h3_apply_condition_noise(sd::Tensor* latent, + const std::shared_ptr& rng) { + auto noise = sd::randn_like(*latent, rng); *latent = *latent * MiniMaxH3::VISUAL_COND_TIMESTEP + - sd::randn_like(*latent, rng) * (1.0f - MiniMaxH3::VISUAL_COND_TIMESTEP); + noise * (1.0f - MiniMaxH3::VISUAL_COND_TIMESTEP); } sd::Tensor h3_pack_audio_and_video_latents(const sd::Tensor& video, @@ -813,12 +869,6 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, if (vision_start_tokens.size() != 1 || vision_end_tokens.size() != 1 || image_pad_tokens.size() != 1) { return set_minimax_error(error, "MiniMax-H3 tokenizer special tokens are invalid"); } - auto append_text = [&](const std::string& text) { - pending_text += text; - if (verify_token_build) { - legacy_presentation += text; - } - }; auto flush_text = [&]() { if (pending_text.empty()) { return; @@ -831,6 +881,13 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, tokens.insert(tokens.end(), text_tokens.begin(), text_tokens.end()); pending_text.clear(); }; + auto append_text = [&](const std::string& text) { + flush_text(); + pending_text = text; + if (verify_token_build) { + legacy_presentation += text; + } + }; auto append_vision = [&](int64_t count) { flush_text(); tokens.push_back(vision_start_tokens[0]); @@ -985,19 +1042,9 @@ bool MiniMaxH3Pipeline::build_text_context(const char* prompt, flush_text(); if (profile != nullptr) profile->text_tokenize_ms += tokenization_ms; if (verify_token_build) { - const auto legacy_tokens = conditioner_->tokenizer->tokenize(legacy_presentation, nullptr, true, 0, 0, false); - if (legacy_tokens != tokens) { - size_t mismatch = 0; - while (mismatch < legacy_tokens.size() && mismatch < tokens.size() && - legacy_tokens[mismatch] == tokens[mismatch]) { - ++mismatch; - } - std::ostringstream message; - message << "MiniMax-H3 direct token build mismatch at " << mismatch - << ": direct=" << tokens.size() << " legacy=" << legacy_tokens.size(); - return set_minimax_error(error, message.str().c_str()); - } - LOG_INFO("MiniMax-H3 direct token build verified: tokens=%zu", tokens.size()); + LOG_INFO("MiniMax-H3 segmented presentation token build: tokens=%zu bytes=%zu", + tokens.size(), + legacy_presentation.size()); } if (tokens.empty()) { return set_minimax_error(error, "MiniMax-H3 prompt tokenization produced no tokens"); @@ -1209,6 +1256,12 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t set_minimax_error(error, "MiniMax-H3 width and height must be positive multiples of 32"); return ED_STATUS_INVALID_ARGUMENT; } + if (static_cast(params->width) * params->height < H3_MIN_GENERATION_PIXELS) { + set_minimax_error(error, + "MiniMax-H3 output canvas must contain at least 65536 pixels (for example 256x256); " + "the official recommended output uses a 768-pixel short edge"); + return ED_STATUS_INVALID_ARGUMENT; + } const int frames = std::max(5, params->frames); if (frames % 17 != 5) { set_minimax_error(error, "MiniMax-H3 frame count must satisfy 17k + 5"); @@ -1351,7 +1404,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t return set_minimax_error(error, "MiniMax-H3 keyframe VAE encode failed"); } sd::Tensor latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, static_cast(resolved_seed)); + h3_apply_condition_noise(&latent, request_rng); h3_trace_tensor((std::string(name) + "_keyframe_latent").c_str(), latent); keyframe_latents.push_back(std::move(latent)); keyframe_indices.push_back(frame_index); @@ -1368,18 +1421,16 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t auto encode_reference_audio = [&](const ed_audio_t& source, int32_t* index) -> bool { if (audio_vae_ == nullptr || source.data == nullptr || source.sample_count == 0 || source.channels == 0 || source.sample_rate == 0) return false; const int64_t prepare_begin = profile_ptr != nullptr ? ggml_time_ms() : 0; - const uint64_t samples = std::max(1, (source.sample_count * 32000ULL + source.sample_rate / 2) / source.sample_rate); + const uint64_t max_source_samples = static_cast( + static_cast(frames) * source.sample_rate / 24.0L); + const uint64_t source_samples = std::max(1, std::min(source.sample_count, max_source_samples)); + const uint64_t samples = std::max(1, (source_samples * 32000ULL + source.sample_rate - 1) / source.sample_rate); sd::Tensor waveform({static_cast(((samples + 799) / 800) * 800), 2, 1, 1}); - for (uint64_t sample = 0; sample < samples; ++sample) { - const double position = static_cast(sample) * source.sample_rate / 32000.0; - const uint64_t first = std::min(static_cast(position), source.sample_count - 1); - const uint64_t second = std::min(first + 1, source.sample_count - 1); - const float fraction = static_cast(position - first); - for (uint32_t channel = 0; channel < 2; ++channel) { - const uint32_t source_channel = source.channels == 1 ? 0 : std::min(channel, source.channels - 1); - const float a = source.data[first * source.channels + source_channel]; - const float b = source.data[second * source.channels + source_channel]; - waveform.index(sample, channel, 0, 0) = std::clamp(a + (b - a) * fraction, -1.f, 1.f); + for (uint32_t channel = 0; channel < 2; ++channel) { + const uint32_t source_channel = source.channels == 1 ? 0 : std::min(channel, source.channels - 1); + const auto resampled = h3_resample_audio_sinc(source, source_samples, source_channel, 32000); + for (uint64_t sample = 0; sample < std::min(samples, resampled.size()); ++sample) { + waveform.index(sample, channel, 0, 0) = resampled[sample]; } } if (profile_ptr != nullptr) profile.reference_audio_prepare_ms += ggml_time_ms() - prepare_begin; @@ -1387,6 +1438,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t auto encoded = audio_vae_->encode(runtime_->n_threads(), waveform); if (profile_ptr != nullptr) profile.reference_audio_vae_encode_ms += ggml_time_ms() - vae_encode_begin; if (encoded.empty()) return false; + h3_trace_tensor("reference_audio_latent", encoded); *index = static_cast(reference_audio_latents.size()); reference_audio_latents.push_back(std::move(encoded)); return true; @@ -1422,7 +1474,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t return ED_STATUS_GENERATION_FAILED; } sd::Tensor latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, static_cast(resolved_seed)); + h3_apply_condition_noise(&latent, request_rng); h3_trace_tensor(("ref_image_" + std::to_string(reference_index) + "_latent").c_str(), latent); const int32_t encoded_image_index = static_cast(reference_latents.size()); reference_latents.push_back(std::move(latent)); @@ -1475,7 +1527,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t if (profile_ptr != nullptr) profile.reference_video_vae_encode_ms += ggml_time_ms() - vae_encode_begin; if (vae_latent.empty()) { set_minimax_error(error, "MiniMax-H3 Ref2VA video VAE encode failed"); return ED_STATUS_GENERATION_FAILED; } auto latent = vae_->vae_to_diffusion_latents(vae_latent); - h3_apply_condition_noise(&latent, static_cast(resolved_seed)); + h3_apply_condition_noise(&latent, request_rng); h3_trace_tensor(("ref_video_" + std::to_string(video_index) + "_latent").c_str(), latent); const int32_t encoded_video_index = static_cast(reference_latents.size()); reference_latents.push_back(std::move(latent)); @@ -1607,12 +1659,11 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t const float sigma = sigma_at(step); const float sigma_next = sigma_at(step + 1); sd::Tensor model_packed = packed; - float audio_sigma = sigma; - float audio_slope = 1.0f; + const float audio_sigma = MiniMaxH3::time_shift_sigma(sigma, video_sigma_shift, 3.0f); + const float audio_sigma_next = MiniMaxH3::time_shift_sigma(sigma_next, video_sigma_shift, 3.0f); + const float audio_slope = MiniMaxH3::time_shift_slope(sigma, video_sigma_shift, 3.0f); float audio_scale = 1.0f; if (sampler == ED_SAMPLER_RES_MULTISTEP) { - audio_sigma = MiniMaxH3::time_shift_sigma(sigma, video_sigma_shift, 3.0f); - audio_slope = MiniMaxH3::time_shift_slope(sigma, video_sigma_shift, 3.0f); audio_scale = video_sigma_shift / 3.0f; auto model_av = diffusion_->split_av_latents(packed, audio_length); model_av.second *= audio_sigma / sigma; @@ -1706,7 +1757,11 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t old_sigma_down = sigma_next; have_old_denoised = true; } else { - packed += velocity * (sigma_next - sigma); + auto packed_av = diffusion_->split_av_latents(packed, audio_length); + auto velocity_av = diffusion_->split_av_latents(velocity, audio_length); + packed_av.first += velocity_av.first * (sigma_next - sigma); + packed_av.second += (velocity_av.second / audio_slope) * (audio_sigma_next - audio_sigma); + packed = h3_pack_audio_and_video_latents(packed_av.first, packed_av.second); } if (h3_trace_enabled()) { h3_trace_tensor(("step_" + std::to_string(step) + "_packed").c_str(), packed);