Skip to content

Commit

Permalink
Merge pull request #186 from PBH-BTN/master
Browse files Browse the repository at this point in the history
merge
  • Loading branch information
Ghost-chu authored Jun 25, 2024
2 parents b1c667f + a33295c commit 683d496
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 116 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import com.ghostchu.peerbanhelper.util.HTTPUtil;
import com.ghostchu.peerbanhelper.util.IPAddressUtil;
import com.ghostchu.peerbanhelper.util.rule.MatchResult;
import com.ghostchu.peerbanhelper.util.rule.matcher.IPBanMatcher;
import com.ghostchu.peerbanhelper.util.rule.matcher.IPMatcher;
import com.github.mizosoft.methanol.MutableRequest;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
Expand Down Expand Up @@ -52,7 +52,7 @@
public class IPBlackRuleList extends AbstractRuleFeatureModule {

final private DatabaseHelper db;
private List<IPBanMatcher> ipBanMatchers;
private List<IPMatcher> ipBanMatchers;
private long checkInterval = 86400000; // 默认24小时检查一次
private ScheduledExecutorService scheduledExecutorService;

Expand Down Expand Up @@ -176,7 +176,6 @@ public SlimMsg updateRule(@NotNull ConfigurationSection rule, IPBanRuleUpdateTyp
File tempFile = new File(dir, "temp_" + ruleFileName);
File ruleFile = new File(dir, ruleFileName);
List<IPAddress> ipAddresses = new ArrayList<>();
List<IPAddress> subnetAddresses = new ArrayList<>();
HTTPUtil.retryableSend(HTTPUtil.getHttpClient(false, null), MutableRequest.GET(url), HttpResponse.BodyHandlers.ofFile(Path.of(tempFile.getPath()))).whenComplete((pathHttpResponse, throwable) -> {
if (throwable != null) {
tempFile.delete();
Expand All @@ -185,8 +184,8 @@ public SlimMsg updateRule(@NotNull ConfigurationSection rule, IPBanRuleUpdateTyp
// 如果一致,但ipBanMatchers没有对应的规则内容,则加载内容
if (ipBanMatchers.stream().noneMatch(ele -> ele.getRuleId().equals(ruleId))) {
try {
fileToIPList(name, ruleFile, ipAddresses, subnetAddresses);
ipBanMatchers.add(new IPBanMatcher(ruleId, name, ipAddresses, subnetAddresses));
fileToIPList(ruleFile, ipAddresses);
ipBanMatchers.add(new IPMatcher(ruleId, name, ipAddresses));
log.warn(Lang.IP_BAN_RULE_USE_CACHE, name);
result.set(new SlimMsg(false, Lang.IP_BAN_RULE_USE_CACHE.replace("{}", name), 500));
} catch (IOException ex) {
Expand All @@ -211,34 +210,31 @@ public SlimMsg updateRule(@NotNull ConfigurationSection rule, IPBanRuleUpdateTyp
int ent_count = 0;
if (!tempHash.equals(ruleHash)) {
// 规则文件不存在或者规则文件与临时文件sha256不一致则需要更新
ent_count = fileToIPList(name, tempFile, ipAddresses, subnetAddresses);
ent_count = fileToIPList(tempFile, ipAddresses);
// 更新后重命名临时文件
ruleFile.delete();
tempFile.renameTo(ruleFile);
} else {
// 如果一致,但ipBanMatchers没有对应的规则内容,则加载内容
if (ipBanMatchers.stream().noneMatch(ele -> ele.getRuleId().equals(ruleId))) {
ent_count = fileToIPList(name, tempFile, ipAddresses, subnetAddresses);
ent_count = fileToIPList(tempFile, ipAddresses);
} else {
log.info(Lang.IP_BAN_RULE_NO_UPDATE, name);
result.set(new SlimMsg(true, Lang.IP_BAN_RULE_NO_UPDATE.replace("{}", name), 200));
}
tempFile.delete();
}
// ip列表或者subnet列表不为空代表需要更新matcher
if (!ipAddresses.isEmpty() || !subnetAddresses.isEmpty()) {
if (ent_count > 0) {
// 如果已经存在则更新,否则添加
ipBanMatchers.stream().filter(ele -> ele.getRuleId().equals(ruleId)).findFirst().ifPresentOrElse(ele -> {
ele.setData(name, ipAddresses, subnetAddresses);
ele.setData(name, ipAddresses);
log.info(Lang.IP_BAN_RULE_UPDATE_SUCCESS, name);
result.set(new SlimMsg(true, Lang.IP_BAN_RULE_UPDATE_SUCCESS.replace("{}", name), 200));
}, () -> {
ipBanMatchers.add(new IPBanMatcher(ruleId, name, ipAddresses, subnetAddresses));
ipBanMatchers.add(new IPMatcher(ruleId, name, ipAddresses));
log.info(Lang.IP_BAN_RULE_LOAD_SUCCESS, name);
result.set(new SlimMsg(true, Lang.IP_BAN_RULE_LOAD_SUCCESS.replace("{}", name), 200));
});
}
if (ent_count > 0) {
// 更新日志
try {
db.insertRuleSubLog(ruleId, ent_count, updateType);
Expand All @@ -262,38 +258,16 @@ public SlimMsg updateRule(@NotNull ConfigurationSection rule, IPBanRuleUpdateTyp

/**
* 读取规则文件并转为IpList
* 其中ipv4网段地址转为精确ip
* 考虑到ipv6分配地址通常是/64,所以ipv6网段不转为精确ip
*
* @param ruleName 规则名称
* @param ruleFile 规则文件
* @param ips 精确ip列表
* @param subnets 网段列表
* @param ips ip列表
* @return 加载的行数
*/
private int fileToIPList(String ruleName, File ruleFile, List<IPAddress> ips, List<IPAddress> subnets) throws IOException {
private int fileToIPList(File ruleFile, List<IPAddress> ips) throws IOException {
AtomicInteger count = new AtomicInteger();
Files.readLines(ruleFile, StandardCharsets.UTF_8).forEach(ele -> {
count.getAndIncrement();
IPAddress ipAddress = IPAddressUtil.getIPAddress(ele);
// 判断是否是网段
List<IPAddress> ipsList = new ArrayList<>();
if (null != ipAddress.getNetworkPrefixLength()) {
if (ipAddress.isIPv4() && ipAddress.getNetworkPrefixLength() >= 20) {
// 前缀长度 >= 20 的ipv4网段地址转为精确ip
ipAddress.nonZeroHostIterator().forEachRemaining(ipsList::add);
} else {
subnets.add(ipAddress);
log.debug(Lang.IP_BAN_RULE_LOAD_CIDR, ruleName, ipAddress);
}
} else {
ipsList.add(ipAddress);
}
ipsList.forEach(ip -> {
ip = ip.withoutPrefixLength();
ips.add(ip);
log.debug(Lang.IP_BAN_RULE_LOAD_IP, ruleName, ip);
});
ips.add(IPAddressUtil.getIPAddress(ele));
});
return count.get();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package com.ghostchu.peerbanhelper.util.rule;

import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;

import java.util.List;
import java.util.Map;

/**
* 规则Matcher
*/
@Getter
@Slf4j
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
public abstract class RuleMatcher<T> extends AbstractMatcher {

private final String ruleId;

@Setter
private String ruleName;

public RuleMatcher(String ruleId, String ruleName, List<T> ruleData) {
this.ruleId = ruleId;
setData(ruleName, ruleData);
}

public abstract void setData(String ruleName, List<T> ruleData);

public Map<String, Object> metadata() {
return Map.of("id", ruleId, "rule", ruleName);
}

}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package com.ghostchu.peerbanhelper.util.rule.matcher;

import com.ghostchu.peerbanhelper.text.Lang;
import com.ghostchu.peerbanhelper.util.IPAddressUtil;
import com.ghostchu.peerbanhelper.util.rule.MatchResult;
import com.ghostchu.peerbanhelper.util.rule.RuleMatcher;
import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import inet.ipaddr.IPAddress;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;

import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

@Slf4j
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
public class IPMatcher extends RuleMatcher<IPAddress> {

private List<IPAddress> subnets;
private List<IPAddress> ips;
private BloomFilter<String> bloomFilter;

public IPMatcher(String ruleId, String ruleName, List<IPAddress> ruleData) {
super(ruleId, ruleName, ruleData);
}

/**
* 设置数据
* 其中ipv4网段地址转为精确ip
* 考虑到ipv6分配地址通常是/64,所以ipv6网段不转为精确ip
*
* @param ruleName 规则名
* @param ruleData 规则数据
*/
public void setData(String ruleName, List<IPAddress> ruleData) {
this.ips = new ArrayList<>();
this.subnets = new ArrayList<>();
ruleData.forEach(ipAddress -> {
// 判断是否是网段
List<IPAddress> ipsList = new ArrayList<>();
if (null != ipAddress.getNetworkPrefixLength()) {
if (ipAddress.isIPv4() && ipAddress.getNetworkPrefixLength() >= 20) {
// 前缀长度 >= 20 的ipv4网段地址转为精确ip
ipAddress.nonZeroHostIterator().forEachRemaining(ipsList::add);
} else {
this.subnets.add(ipAddress);
log.debug(Lang.IP_BAN_RULE_LOAD_CIDR, ruleName, ipAddress);
}
} else {
ipsList.add(ipAddress);
}
ipsList.forEach(ip -> {
ip = ip.withoutPrefixLength();
this.ips.add(ip);
log.debug(Lang.IP_BAN_RULE_LOAD_IP, ruleName, ip);
});
});
bloomFilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), this.ips.size(), 0.01);
this.ips.forEach(ip -> bloomFilter.put(ip.toString()));
}

@Override
public @NotNull MatchResult match0(@NotNull String content) {
final IPAddress ip = IPAddressUtil.getIPAddress(content);
// 先用bloom过滤器查一下
if (bloomFilter.mightContain(content)) {
// 如果查到了,那么进一步验证到底是不是在黑名单中(bloom filter存在误报的可能性)
if (ips.stream().anyMatch(ele -> ele.isIPv4Convertible() == ip.isIPv4Convertible() && ele.equals(ip))) {
return MatchResult.TRUE;
}
}
// 最后subnet表查一下
if (subnets.stream().anyMatch(subnet -> subnet.contains(ip))) {
return MatchResult.TRUE;
}
return MatchResult.DEFAULT;
}

@Override
public @NotNull String matcherName() {
return Lang.RULE_MATCHER_SUB_RULE;
}

@Override
public String matcherIdentifier() {
return "peerbanhelper:ipmatcher";
}
}

0 comments on commit 683d496

Please sign in to comment.