Skip to content

fix:remove mmdb #939

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
samwafgo merged 1 commit into from
Aug 12, 2026
Merged

fix:remove mmdb #939

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 73 additions & 3 deletions api/waf_iplocation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"net"
"os"
"path/filepath"
"strings"

"github.com/gin-gonic/gin"
)
Expand Down Expand Up @@ -189,7 +190,7 @@ func (w *WafIPLocationApi) GetIPDBConfigApi(c *gin.Context) {
resp := IPDBConfigResp{
Ipv4Source: getConfigOrDefault("ip_v4_source", "ip2region"),
Ipv4Format: getConfigOrDefault("ip_v4_format", "legacy"),
Ipv6Source: getConfigOrDefault("ip_v6_source", "geolite2"),
Ipv6Source: getConfigOrDefault("ip_v6_source", "ip2region"),
Ipv6Format: getConfigOrDefault("ip_v6_format", "legacy"),
}
response.OkWithDetailed(resp, "获取成功", c)
Expand Down Expand Up @@ -259,7 +260,14 @@ func (w *WafIPLocationApi) SaveIPDBConfigApi(c *gin.Context) {

// UploadIPDBFileApi 上传 IP 数据库文件
func (w *WafIPLocationApi) UploadIPDBFileApi(c *gin.Context) {
// key 是界面上那张表的槽位标识。带了 key 就按槽位定义确定落盘文件名与允许后缀，
// 不用再靠扩展名猜用户想传哪个库（.mmdb 到底是 IPv4 还是 IPv6 的，靠猜是猜不准的）。
// 不带 key 时退回老的 type+扩展名逻辑，保证老前端/脚本还能用。
slotKey := c.PostForm("key")
ipType := c.PostForm("type")
if slot, ok := iplocation.DatabaseByKey(slotKey); ok && ipType == "" {
ipType = slot.UploadType
}
if ipType != "ipv4" && ipType != "ipv6" && ipType != "ipdb" {
response.FailWithMessage("无效的类型参数，必须是 ipv4、ipv6 或 ipdb", c)
return
Expand All @@ -276,6 +284,11 @@ func (w *WafIPLocationApi) UploadIPDBFileApi(c *gin.Context) {
response.FailWithMessage("不支持的文件类型，仅支持 .xdb、.mmdb 和 .ipdb 文件", c)
return
}
// 带了 key 就用槽位声明的后缀严格卡一遍，别让用户把 mmdb 传进 ip2region 的槽
if slot, ok := iplocation.DatabaseByKey(slotKey); ok && !strings.EqualFold(ext, slot.Accept) {
response.FailWithMessage(fmt.Sprintf("【%s】只接受 %s 文件，请确认后重新上传", slot.Desc, slot.Accept), c)
return
}
if ext == ".ipdb" && ipType != "ipdb" {
response.FailWithMessage(".ipdb 文件请使用 type=ipdb 上传", c)
return
Expand Down Expand Up @@ -329,9 +342,11 @@ func (w *WafIPLocationApi) UploadIPDBFileApi(c *gin.Context) {
return
}

// xdb / mmdb 保存路径
// xdb / mmdb 保存路径：优先按槽位定义取，退回老的 type+扩展名推断
var finalPath string
if ipType == "ipv4" {
if slot, ok := iplocation.DatabaseByKey(slotKey); ok {
finalPath = filepath.Join(dataDir, slot.FileName)
} else if ipType == "ipv4" {
if ext == ".xdb" {
finalPath = filepath.Join(dataDir, "ip2region.xdb")
} else {
Expand Down Expand Up @@ -412,6 +427,61 @@ func (w *WafIPLocationApi) ReloadIPDBApi(c *gin.Context) {
response.OkWithMessage("数据库重新加载成功", c)
}

// CheckIPDBUpgradeApi 检查可在线下载的 IP 数据库
//
// 只有 ip2region 系列（Apache-2.0）能由 SamWaf 转发分发；GeoLite2 与 ipdb 受各自授权限制，
// 只能由用户自行获取后上传。
func (w *WafIPLocationApi) CheckIPDBUpgradeApi(c *gin.Context) {
dataDir := filepath.Join(utils.GetCurrentDir(), "data")
info, err := iplocation.CheckUpgrade(dataDir)
if err != nil {
// 拿不到远端清单也要把本地状态回给前端（内网环境下这是常态），
// 所以带着 info 一起返回，让界面能显示"本地已有哪些库"。
response.OkWithDetailed(map[string]interface{}{
"info": info,
"error": err.Error(),
}, "获取远端升级信息失败，仅返回本地状态", c)
return
}
response.OkWithDetailed(map[string]interface{}{"info": info}, "获取成功", c)
}

// ApplyIPDBUpgradeApi 启动下载指定的 IP 数据库（异步），下载完成后自动热加载
//
// IPv6 库约 35MB，慢网要几分钟，同步等会被网关掐断、用户也只能看个转圈。
// 这里立刻返回，前端轮询 GetIPDBUpgradeProgressApi 画进度条。
func (w *WafIPLocationApi) ApplyIPDBUpgradeApi(c *gin.Context) {
var req struct {
Key string `json:"key" binding:"required"`
}
if err := c.ShouldBindJSON(&req); err != nil {
response.FailWithMessage("参数解析失败", c)
return
}
dataDir := filepath.Join(utils.GetCurrentDir(), "data")
if err := iplocation.StartUpgrade(dataDir, req.Key, reloadManagerByCurrentConfig); err != nil {
response.FailWithMessage(err.Error(), c)
return
}
response.OkWithMessage("已开始下载", c)
}

// GetIPDBUpgradeProgressApi 查询当前下载进度，供前端轮询
func (w *WafIPLocationApi) GetIPDBUpgradeProgressApi(c *gin.Context) {
response.OkWithDetailed(iplocation.GetProgress(), "获取成功", c)
}

// CancelIPDBUpgradeApi 取消正在进行的下载
//
// 官方源在部分网络下速度很差，用户等不下去时要能停，转而自己从 Gitee/GitHub 下好丢进 data 目录。
func (w *WafIPLocationApi) CancelIPDBUpgradeApi(c *gin.Context) {
if err := iplocation.CancelUpgrade(); err != nil {
response.FailWithMessage(err.Error(), c)
return
}
response.OkWithMessage("已取消下载", c)
}

// TestIPLookupApi 测试 IP 查询
func (w *WafIPLocationApi) TestIPLookupApi(c *gin.Context) {
var req struct {
Expand Down
Binary file removed cmd/samwaf/exedata/GeoLite2-Country.mmdb
View file
Open in desktop
Binary file not shown.
179 changes: 49 additions & 130 deletions cmd/samwaf/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ import (
"embed"
_ "embed"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
Expand All @@ -68,8 +67,9 @@ import (
//go:embed exedata/ip2region.xdb
var Ip2regionBytes []byte // 当前目录，解析为[]byte类型

//go:embed exedata/GeoLite2-Country.mmdb
var Ipv6CountryBytes []byte // IPv6国家解析
// GeoLite2-Country.mmdb 自 1.3.24-beta.6 起不再内嵌，
// 把它编进发行的二进制等同于再分发。需要 GeoLite2 的用户可自行到 MaxMind 官网下载 mmdb
// 放进 data/ 目录，程序会照常加载。IPv6 默认改用 ip2region（Apache-2.0，可自由分发）。

//go:embed exedata/ldpconfig.yml
var ldpConfig string //隐私防护ldp
Expand Down Expand Up @@ -159,136 +159,30 @@ func (m *wafSystenService) run() {
//初始化步骤[加载ip数据库]
// 创建 IP Location Manager
global.GIPLOCATION_MANAGER = iplocation.NewManager()
// 注册内置数据库，供 manager 在磁盘无文件时兜底加载
iplocation.SetBuiltinData(Ip2regionBytes, Ipv6CountryBytes)

// 根据配置加载 IPv4 数据库：每种来源读各自的文件，磁盘无文件时回落到内置数据
if global.GCONFIG_IP_V4_SOURCE == "ip2region" {
ip2RegionFilePath := filepath.Join(utils.GetCurrentDir(), "data", "ip2region.xdb")
var ipv4Data []byte
ipv4FromBuiltin := false
if _, err := os.Stat(ip2RegionFilePath); os.IsNotExist(err) {
// 使用内置数据
ipv4Data = Ip2regionBytes
ipv4FromBuiltin = true
zlog.Info("Using embedded IPv4 ip2region database, size: ", len(ipv4Data))
} else {
// 读取外部文件
fileBytes, err := ioutil.ReadFile(ip2RegionFilePath)
if err != nil {
log.Fatalf("Failed to read IP database file ip2region.xdb: %v", err)
}
ipv4Data = fileBytes
zlog.Info("IPv4 database ip2region.xdb loaded from file, size: ", len(ipv4Data), ip2RegionFilePath)
}

err := global.GIPLOCATION_MANAGER.LoadV4Ip2Region(ipv4Data, iplocation.DBFormat(global.GCONFIG_IP_V4_FORMAT))
if err != nil {
log.Fatalf("Failed to load IPv4 ip2region database: %v", err)
}
global.GIPLOCATION_MANAGER.SetV4Builtin(ipv4FromBuiltin)
zlog.Info("IPv4 ip2region database loaded successfully")
} else if global.GCONFIG_IP_V4_SOURCE == "geolite2" {
// GeoLite2 读的是 mmdb，与 ip2region.xdb 是两种格式，不能混用同一份字节
ipv4GeoLitePath := filepath.Join(utils.GetCurrentDir(), "data", "GeoLite2-Country.mmdb")
var ipv4Data []byte
ipv4FromBuiltin := false
if _, err := os.Stat(ipv4GeoLitePath); os.IsNotExist(err) {
// 内置 GeoLite2-Country.mmdb，IPv4/IPv6 共用同一份
ipv4Data = Ipv6CountryBytes
ipv4FromBuiltin = true
zlog.Info("Using embedded IPv4 GeoLite2 database, size: ", len(ipv4Data))
} else {
fileBytes, err := ioutil.ReadFile(ipv4GeoLitePath)
if err != nil {
log.Fatalf("Failed to read IPv4 GeoLite2 database file: %v", err)
}
ipv4Data = fileBytes
zlog.Info("IPv4 GeoLite2 database loaded from file, size: ", len(ipv4Data), ipv4GeoLitePath)
}

err := global.GIPLOCATION_MANAGER.LoadV4GeoLite2(ipv4Data)
if err != nil {
log.Fatalf("Failed to load IPv4 GeoLite2 database: %v", err)
}
global.GIPLOCATION_MANAGER.SetV4Builtin(ipv4FromBuiltin)
zlog.Info("IPv4 GeoLite2 database loaded successfully")
} else if global.GCONFIG_IP_V4_SOURCE == "ipdb" {
ipdbPath := filepath.Join(utils.GetCurrentDir(), "data", "iplocation.ipdb")
if _, err := os.Stat(ipdbPath); err == nil {
if err2 := global.GIPLOCATION_MANAGER.LoadIpdb(ipdbPath); err2 != nil {
zlog.Warn("Failed to load ipdb database (v4): ", err2)
} else {
zlog.Info("ipdb database loaded successfully (v4 source)")
}
} else {
zlog.Warn("ipdb database file not found, please upload iplocation.ipdb")
}
// 注册内置数据库，供 manager 在磁盘无文件时兜底加载。
// GeoLite2 已去内嵌，这里只剩 IPv4 的 ip2region 一份兜底。
iplocation.SetBuiltinData(Ip2regionBytes, nil)

// 加载优先级与热重载共用同一份实现（磁盘文件 > 内置数据 > 同类型其它来源降级）。
// IP 库属于「有更好、没有也得能跑」的辅助数据：任何加载失败都只告警，
// 绝不能阻止 WAF 启动——防护能力不依赖地区库。
if err := global.GIPLOCATION_MANAGER.ReloadFromConfig(
filepath.Join(utils.GetCurrentDir(), "data"),
global.GCONFIG_IP_V4_SOURCE, global.GCONFIG_IP_V6_SOURCE,
global.GCONFIG_IP_V4_FORMAT, global.GCONFIG_IP_V6_FORMAT,
); err != nil {
zlog.Warn("IP地理位置数据库加载失败，地区相关功能将不可用: ", err.Error())
}

// 加载 IPv6 数据库
if global.GCONFIG_IP_V6_SOURCE == "ip2region" {
// IPv6 ip2region 需要单独的文件
ipv6Ip2RegionPath := filepath.Join(utils.GetCurrentDir(), "data", "ip2region_v6.xdb")
if _, err := os.Stat(ipv6Ip2RegionPath); err == nil {
fileBytes, err := ioutil.ReadFile(ipv6Ip2RegionPath)
if err != nil {
zlog.Warn("Failed to read IPv6 ip2region database file: ", err)
} else {
err = global.GIPLOCATION_MANAGER.LoadV6Ip2Region(fileBytes, iplocation.DBFormat(global.GCONFIG_IP_V6_FORMAT))
if err != nil {
zlog.Warn("Failed to load IPv6 ip2region database: ", err)
} else {
global.GIPLOCATION_MANAGER.SetV6Builtin(false)
zlog.Info("IPv6 ip2region database loaded successfully, size: ", len(fileBytes))
}
}
} else {
zlog.Warn("IPv6 ip2region database file not found, please upload ip2region_v6.xdb")
}
} else if global.GCONFIG_IP_V6_SOURCE == "geolite2" {
// IPv6 GeoLite2
ipv6GeoLitePath := filepath.Join(utils.GetCurrentDir(), "data", "GeoLite2-Country.mmdb")
var ipv6Data []byte
ipv6FromBuiltin := false
if _, err := os.Stat(ipv6GeoLitePath); os.IsNotExist(err) {
// 使用内置数据
ipv6Data = Ipv6CountryBytes
ipv6FromBuiltin = true
zlog.Info("Using embedded IPv6 GeoLite2 database, size: ", len(ipv6Data))
} else {
// 读取外部文件
fileBytes, err := ioutil.ReadFile(ipv6GeoLitePath)
if err != nil {
log.Fatalf("Failed to read IPv6 GeoLite2 database file: %v", err)
}
ipv6Data = fileBytes
zlog.Info("IPv6 GeoLite2 database loaded from file, size: ", len(ipv6Data), ipv6GeoLitePath)
}

err := global.GIPLOCATION_MANAGER.LoadV6GeoLite2(ipv6Data)
if err != nil {
log.Fatalf("Failed to load IPv6 GeoLite2 database: %v", err)
}
global.GIPLOCATION_MANAGER.SetV6Builtin(ipv6FromBuiltin)
zlog.Info("IPv6 GeoLite2 database loaded successfully")
} else if global.GCONFIG_IP_V6_SOURCE == "ipdb" {
// 如果 v4 已经加载了 ipdb，跳过重复加载
if !global.GIPLOCATION_MANAGER.IsIpdbLoaded() {
ipdbPath := filepath.Join(utils.GetCurrentDir(), "data", "iplocation.ipdb")
if _, err := os.Stat(ipdbPath); err == nil {
if err2 := global.GIPLOCATION_MANAGER.LoadIpdb(ipdbPath); err2 != nil {
zlog.Warn("Failed to load ipdb database (v6): ", err2)
} else {
zlog.Info("ipdb database loaded successfully (v6 source)")
}
} else {
zlog.Warn("ipdb database file not found, please upload iplocation.ipdb")
}
} else {
zlog.Info("ipdb database already loaded (shared with v4)")
if st := global.GIPLOCATION_MANAGER.GetStatus(); st != nil {
zlog.Info(fmt.Sprintf("IP库加载完成 IPv4:%s(内置:%v,%d字节) IPv6:%s(内置:%v,%d字节)",
st.IPv4Source, st.IPv4Builtin, st.IPv4FileSize,
st.IPv6Source, st.IPv6Builtin, st.IPv6FileSize))
if st.IPv6FileSize == 0 {
zlog.Warn("IPv6 地区库不可用：IPv6 访客的归属地将显示为未知，且地区类自定义规则对 IPv6 请求不生效。" +
"可在【IP库管理】里在线下载，或自行下载 ip2region_v6.xdb 放入 data/ 目录")
}
}

global.GWAF_DLP_CONFIG = ldpConfig
global.GWAF_REG_PUBLIC_KEY = publicKey

Expand Down Expand Up @@ -355,6 +249,31 @@ func (m *wafSystenService) run() {
// 创建 Snowflake 实例
global.GWAF_SNOWFLAKE_GEN = wafsnowflake.NewSnowflake(1609459200000, 1, 1) // 设置epoch时间、机器ID和数据中心ID

// 注入 IP 库在线下载上下文：iplocation 不能反向依赖 global/utils（会成环），
// 所以升级源、SSRF 安全客户端、通知回调都从这里传进去。
iplocation.ConfigureUpgrader(iplocation.UpgradeConfig{
UpdateVersionURL: global.GUPDATE_VERSION_URL,
NewClient: utils.SafeHTTPClient,
ValidateURL: utils.IsSafeOutboundURL,
NotifyFunc: func(success bool, msg string) {
if global.GQEQUE_MESSAGE_DB == nil {
return
}
successStr := "false"
if success {
successStr = "true"
}
global.GQEQUE_MESSAGE_DB.Enqueue(innerbean.UpdateResultMessageInfo{
BaseMessageInfo: innerbean.BaseMessageInfo{
OperaType: "IP库下载",
Server: global.GWAF_CUSTOM_SERVER_NAME,
},
Msg: msg,
Success: successStr,
})
},
})

// 创建owasp 管理器（支持热重载）
global.GWAF_OWASP_MANAGER = wafowasp.NewOwaspManager(utils.GetCurrentDir())
global.GWAF_OWASP = global.GWAF_OWASP_MANAGER.Current()
Expand Down
Loading
Loading