This commit is contained in:
2024-01-11 22:30:08 +08:00
parent 3f80716092
commit a177d5b2cf
9 changed files with 403 additions and 128 deletions

View File

@@ -35,6 +35,7 @@ public class Constants {
public static final String CACHE_ADDRESS_TYPE = "addressType";
public static final String CACHE_CLIENT_RANDOM_PREFIX = "clientRandom::";
public static final String HTTP_HEADER_RESPONSE_TIME = "X-Response-Time";
public static final String SYSCONFIG_REQUEST_SIGN_IGNORE_URLS = "request_sign_ignore_urls";
public static String SYSCONFIG_ENABLE_CAPTCHA = "enable_captcha";
public static String SYSCONFIG_ENABLE_REQUEST_SIGN = "enable_request_sign";
public static String SYSCONFIG_SESSION_TIMEOUT = "session_timeout";

View File

@@ -0,0 +1,38 @@
package cn.lihongjie.coal.common;
import lombok.experimental.UtilityClass;
import java.text.CharacterIterator;
import java.text.StringCharacterIterator;
import java.util.*;
@UtilityClass
public class FormatUtils {
public static String humanReadableByteCountSI(long bytes) {
if (-1000 < bytes && bytes < 1000) {
return bytes + " B";
}
CharacterIterator ci = new StringCharacterIterator("kMGTPE");
while (bytes <= -999_950 || bytes >= 999_950) {
bytes /= 1000;
ci.next();
}
return String.format("%.1f %cB", bytes / 1000.0, ci.current());
}
public static String humanReadableByteCountBin(long bytes) {
long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes);
if (absB < 1024) {
return bytes + " B";
}
long value = absB;
CharacterIterator ci = new StringCharacterIterator("KMGTPE");
for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10) {
value >>= 10;
ci.next();
}
value *= Long.signum(bytes);
return String.format("%.1f %ciB", value / 1024.0, ci.current());
}
}

View File

@@ -2,11 +2,15 @@ package cn.lihongjie.coal.common;
import cn.lihongjie.coal.exception.BizException;
import io.vavr.Function2;
import lombok.Data;
import lombok.experimental.UtilityClass;
import org.apache.poi.ss.formula.functions.T;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
@UtilityClass
@@ -40,4 +44,73 @@ public class TreeUtils {
}
return ans;
}
public static <T>void dfs(T src, Function<T, Iterable<T>> getter, Consumer<DFSCtx<T>> handler, Function<T, Object> hashFn){
var seen = new HashSet<>();
Stack<DFSCtx<T>> stack = new Stack<>();
stack.push(new DFSCtx<>(src, null, src));
while (!stack.isEmpty()) {
DFSCtx<T> pop = stack.pop();
if (hashFn!=null && !seen.add(hashFn.apply(pop.current))) {
throw new BizException("循环引用 " + pop.current.toString());
}
handler.accept(pop);
Iterable<T> children = getter.apply(pop.current);
if (children != null) {
for (T child : children) {
stack.push(new DFSCtx<>(src, pop.current, child));
}
}
}
}
public static <T> void forEachChildren(
T src,
Function<T, Iterable<T>> getter,
Function2<T, Iterable<T>, Void> setter,
Function<Iterable<T>, Iterable<T>> mapper) {
var seen = new HashSet<>();
Stack<T> stack = new Stack<>();
stack.push(src);
while (!stack.isEmpty()) {
T pop = stack.pop();
// if (!seen.add(pop)) {
// throw new BizException("循环引用 " + pop.toString());
// }
Iterable<T> children = getter.apply(pop);
if (children != null) {
for (T child : children) {
stack.push(child);
}
setter.apply(pop, mapper.apply( children));
}
}
}
@Data
public static class DFSCtx<T>{
private T root;
private T parent;
private T current;
public DFSCtx(T root, T parent, T current) {
this.root = root;
this.parent = parent;
this.current = current;
}
}
}

View File

@@ -28,6 +28,7 @@ import org.springframework.core.annotation.Order;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.util.AntPathMatcher;
import org.springframework.util.StopWatch;
import org.springframework.web.filter.OncePerRequestFilter;
@@ -66,6 +67,21 @@ public class SignFilter extends OncePerRequestFilter {
doFilter(request, response, filterChain);
return;
}
String configVal =
sysConfigService.getConfigVal(Constants.SYSCONFIG_REQUEST_SIGN_IGNORE_URLS);
if (StringUtils.isNotEmpty(configVal)) {
AntPathMatcher antPathMatcher = new AntPathMatcher();
String[] ignoreUrls = configVal.split(",");
for (String ignoreUrl : ignoreUrls) {
if (antPathMatcher.match(ignoreUrl, request.getRequestURI())) {
doFilter(request, response, filterChain);
return;
}
}
}
StopWatch stopWatch = new StopWatch();
stopWatch.start("基本请求头");
@@ -188,8 +204,7 @@ public class SignFilter extends OncePerRequestFilter {
}
stopWatch.start("重复请求");
String cr =
redisTemplate.opsForValue().get(Constants.CACHE_CLIENT_RANDOM_PREFIX + random);
String cr = redisTemplate.opsForValue().get(Constants.CACHE_CLIENT_RANDOM_PREFIX + random);
if (StringUtils.equals(cr, "1")) {
sysLogService.saveSysLog(request, "签名模块", "重复请求", "");

View File

@@ -8,6 +8,8 @@ import cn.lihongjie.coal.base.dto.R;
import cn.lihongjie.coal.netDisk.dto.*;
import cn.lihongjie.coal.netDisk.service.NetDiskService;
import com.google.common.base.Splitter;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
@@ -56,7 +58,6 @@ public class NetDiskController {
return R.success(this.service.idToPath(request));
}
/**
* id转路径, 同时返回所有父级详细信息
*
@@ -68,7 +69,6 @@ public class NetDiskController {
return R.success(this.service.idToPath2(request));
}
/**
* 路径转id
*
@@ -87,7 +87,7 @@ public class NetDiskController {
* @return
*/
@PostMapping("/ls")
@SysLog(action = "ls", message = "id")
// @SysLog(action = "ls", message = "id")
public List<NetDiskDto> ls(@RequestBody IdRequest request) {
return this.service.ls(request);
}
@@ -102,6 +102,7 @@ public class NetDiskController {
public NetDiskTreeDto tree(@RequestBody IdRequest request) {
return this.service.tree(request);
}
/**
* 列出文件夹树
*
@@ -297,16 +298,18 @@ public class NetDiskController {
}
/**
* 通过阿里云OSS链接下载文件夹
* 批量下载文件/文件夹
*
* @param request
* @param response
*/
@PostMapping("/downloadDir")
@SysLog(action = "downloadDir", message = "id")
public void downloadDir(@RequestBody IdRequest request, HttpServletResponse response) {
@GetMapping("/downloadBatch")
@SysLog(action = "downloadBatch", message = "ids")
public void downloadDir(@RequestParam("ids") String ids, HttpServletResponse response) {
this.service.downloadDir(request, response);
this.service.downloadBatch(
new IdRequest(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(ids)),
response);
}
/**

View File

@@ -1,11 +1,9 @@
package cn.lihongjie.coal.netDisk.entity;
import cn.lihongjie.coal.base.entity.OrgCommonEntity;
import cn.lihongjie.coal.common.FormatUtils;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Entity;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.*;
import lombok.Data;
@@ -15,10 +13,10 @@ import org.hibernate.annotations.Formula;
import java.util.List;
@Data
@Entity
@Entity()
public class NetDiskEntity extends OrgCommonEntity {
@ManyToOne
@ManyToOne(fetch = FetchType.LAZY)
private NetDiskEntity parent;
@Comment("条目类型, 文件夹/目录/分片文件")
@@ -33,50 +31,53 @@ public class NetDiskEntity extends OrgCommonEntity {
+ " and i.code = entry_type)")
private String entryTypeName;
@OneToMany(mappedBy = "parent", cascade = {CascadeType.REMOVE})
@OneToMany(
mappedBy = "parent",
cascade = {CascadeType.REMOVE})
private List<NetDiskEntity> children;
@Comment("文件/文件夹 大小, 单位 byte")
@Access(AccessType.PROPERTY)
private Long size;
// @Formula("(pg_size_pretty(size))")
@Transient private String sizeHumanReadable;
@Formula("(pg_size_pretty(size))")
private String sizeHumanReadable;
public void setSize(Long size) {
this.size = size;
try {
this.sizeHumanReadable = FormatUtils.humanReadableByteCountBin(size);
} catch (Exception e) {
}
}
@Comment("文件 sha256 值, 文件夹没有")
private String sha256;
@Comment("文件 mime 类型, 文件夹没有")
private String mimeType;
private String ossKey;
@Comment("分片序号")
private Long sliceIndex;
@Comment("分片 sha256 值")
private String sliceSha256;
@Comment("分片文件大小")
private Long sliceSize;
@Comment("分片上传任务ID")
private String sliceTaskId;
@Override
public void prePersist() {
super.prePersist();
if (this.size == null){
if (this.size == null) {
this.size = 0L;
}
}

View File

@@ -108,4 +108,20 @@ select max(depth) from tmp
""", nativeQuery = true)
Integer dirDepth(@Param("id") String id);
@Query(value = """
with recursive tmp as (
select 0 as depth, t.id as id from t_net_disk t where t.id = :id
union all
select tmp.depth + 1 as depth, t.id as id from t_net_disk t
inner join tmp on t.parent_id = tmp.id
where t.entry_type = '0'
)
select id from tmp
""", nativeQuery = true)
List<String> findAllSubDir(@Param("id") String id );
}

View File

@@ -18,6 +18,7 @@ import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.*;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Splitter;
import jakarta.persistence.EntityManager;
import jakarta.persistence.criteria.CriteriaBuilder;
@@ -52,6 +53,7 @@ import java.io.InputStream;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Supplier;
@@ -184,36 +186,78 @@ public class NetDiskService extends BaseService<NetDiskEntity, NetDiskRepository
return this.mapper.toDto(ans);
}
@Autowired private EntityManager entityManager;
public List<NetDiskDto> batchCreateDir(BatchCreateDirDto request) {
request.getNames().forEach(x -> validateDirName(x));
List<String> paths = request.getNames();
if (StringUtils.isEmpty(request.getParent())) {
ArrayList<NetDiskDto> ans = new ArrayList<>();
for (Object o : request.getNames()) {
String name = (String) o;
NetDiskDto dto = createDir(new CreateDirDto(request.getParent(), name));
ans.add(dto);
paths.forEach(
x -> {
if (!x.startsWith("/")) {
throw new BizException("根目录下的文件夹必须以 / 开头");
}
});
batchCreateDir(
findRoot(),
paths.stream().map(s -> s.substring(1)).collect(Collectors.toList()));
} else {
NetDiskEntity parent = get(request.getParent());
paths.forEach(
x -> {
if (x.startsWith("/")) {
throw new BizException("子目录下的文件夹不能以 / 开头");
}
});
batchCreateDir(parent, paths);
}
return ans;
return null;
}
private void handlerRootDir(IdRequest request) {
if (StringUtils.equals(request.getId(), "/")) {
private void batchCreateDir(NetDiskEntity parent, List<String> paths) {
List<NetDiskEntity> rootDir = findRootDir();
paths.forEach(
x ->
batchCreateDir0(
parent,
Splitter.on("/").trimResults().omitEmptyStrings().splitToList(x)));
}
if (rootDir.isEmpty()) {
NetDiskEntity entity = new NetDiskEntity();
entity.setEntryType("0");
entity.setName("/");
entity.setOrganizationId(Ctx.currentUser().getOrganizationId());
save(entity);
private void batchCreateDir0(NetDiskEntity parent, List<String> path) {
request.setId(entity.getId());
} else {
request.setId(rootDir.get(0).getId());
}
if (path.isEmpty()) {
return;
}
String first = path.get(0);
if (parent.getChildren() == null
|| parent.getChildren().stream().noneMatch(x -> x.getName().equals(first))) {
NetDiskEntity entity = new NetDiskEntity();
entity.setName(first);
entity.setEntryType("0");
entity.setOrganizationId(Ctx.currentUser().getOrganizationId());
entity.setParent(parent);
save(entity);
parent.getChildren().add(entity);
batchCreateDir0(entity, path.subList(1, path.size()));
} else {
batchCreateDir0(
parent.getChildren().stream()
.filter(x -> x.getName().equals(first))
.findFirst()
.get(),
path.subList(1, path.size()));
}
}
@@ -625,35 +669,24 @@ public class NetDiskService extends BaseService<NetDiskEntity, NetDiskRepository
return null;
}
public NetDiskDto copy(CopyDto request) {
private void handlerRootDir(IdRequest request) {
NetDiskEntity target = get(request.getTargetId());
if (StringUtils.equals(request.getId(), "/")) {
if (!target.getEntryType().equals("0")) {
throw new BizException("只能复制到目录");
}
List<NetDiskEntity> rootDir = findRootDir();
for (String srcId : request.getSrcIds()) {
if (rootDir.isEmpty()) {
NetDiskEntity entity = new NetDiskEntity();
entity.setEntryType("0");
entity.setName("/");
entity.setOrganizationId(Ctx.currentUser().getOrganizationId());
save(entity);
NetDiskEntity srcEntity = get(srcId);
NetDiskEntity src = this.mapper.copyChildren(srcEntity);
if (target.getChildren().stream().filter(x -> x.getName().equals(src.getName())).count()
> 0) {
throw new BizException("目标目录已经存在同名文件/文件夹");
request.setId(entity.getId());
} else {
request.setId(rootDir.get(0).getId());
}
src.setParent(target);
this.save(src);
}
if (target.getParent() != null) {
updateDirSize(target.getParent().getId());
}
return null;
}
public String downloadFile(String id, boolean attachment) {
@@ -694,64 +727,82 @@ public class NetDiskService extends BaseService<NetDiskEntity, NetDiskRepository
return url.toString();
}
@SneakyThrows
public void downloadDir(IdRequest id, HttpServletResponse response) {
public NetDiskDto copy(CopyDto request) {
NetDiskEntity dir = get(id.getId());
NetDiskEntity target = get(request.getTargetId());
if (!dir.getEntryType().equals("0")) {
throw new BizException("只能下载目录");
if (!target.getEntryType().equals("0")) {
throw new BizException("只能复制到目录");
}
StopWatch stopWatch = new StopWatch("下载文件夹 " + dir.getName() + " " + dir.getId());
for (String srcId : request.getSrcIds()) {
NetDiskEntity srcEntity = get(srcId);
NetDiskEntity src = this.mapper.copyChildren(srcEntity);
if (target.getChildren().stream().filter(x -> x.getName().equals(src.getName())).count()
> 0) {
throw new BizException("目标目录已经存在同名文件/文件夹");
}
TreeUtils.dfs(
src,
NetDiskEntity::getChildren,
x -> {
if (x.getRoot() == x.getCurrent() && x.getParent() == null) {
x.getCurrent().setParent(target);
this.save(x.getCurrent());
} else {
x.getCurrent().setParent(x.getParent());
this.save(x.getCurrent());
}
},
null);
}
if (target.getParent() != null) {
updateDirSize(target.getParent().getId());
}
return null;
}
@SneakyThrows
public void downloadBatch(IdRequest request, HttpServletResponse response) {
// 设置响应类型为ZIP
response.setContentType("application/zip");
// 设置响应头指定ZIP文件名
String zipName = URLEncoder.encode(dir.getName() + ".zip", StandardCharsets.UTF_8);
String zipName = URLEncoder.encode("批量下载" + LocalDate.now() + ".zip", StandardCharsets.UTF_8);
response.setHeader("Content-Disposition", "attachment; filename=" + zipName);
StopWatch stopWatch = new StopWatch("批量下载 "+ StringUtils.join(request.getIds(), ","));
@Cleanup("close")
ZipArchiveOutputStream zipArchiveOutputStream =
new ZipArchiveOutputStream(response.getOutputStream());
zipDir(dir, zipArchiveOutputStream, dir, stopWatch);
for (String id : request.getIds()) {
log.info(stopWatch.prettyPrint());
}
NetDiskEntity entity = get(id);
@SneakyThrows
private void zipDir(
NetDiskEntity dir,
ZipArchiveOutputStream zipArchiveOutputStream,
NetDiskEntity root,
StopWatch stopWatch) {
stopWatch.start(
"下载文件夹/文件夹 "
+ entity.getName()
+ " "
+ entity.getEntryTypeName()
+ " "
+ entity.getId());
if (dir.getEntryType().equals("0")) {
stopWatch.start("创建目录 " + dir.getName() + " " + dir.getId());
zipArchiveOutputStream.putArchiveEntry(new ZipArchiveEntry(getFullName(root, dir)));
zipArchiveOutputStream.closeArchiveEntry();
zipEntity(entity, zipArchiveOutputStream, entity, stopWatch);
stopWatch.stop();
} else {
zipArchiveOutputStream.putArchiveEntry(new ZipArchiveEntry(getFullName(root, dir)));
stopWatch.start("下载文件 " + dir.getName() + " " + dir.getId());
OSSObject ossObject =
ossClient.getObject(aliyunProperty.getOSS().getBucketName(), dir.getOssKey());
IOUtils.copy(ossObject.getObjectContent(), zipArchiveOutputStream);
stopWatch.stop();
zipArchiveOutputStream.closeArchiveEntry();
for (NetDiskEntity child : dir.getChildren()) {
zipDir(child, zipArchiveOutputStream, root, stopWatch);
}
log.info(stopWatch.prettyPrint());
}
}
@@ -902,45 +953,92 @@ public class NetDiskService extends BaseService<NetDiskEntity, NetDiskRepository
public NetDiskTreeDto tree(IdRequest request) {
NetDiskEntity entity = get(request.getId());
return this.mapper.toTreeDto(entity);
}
@Autowired
private EntityManager entityManager;
@SneakyThrows
private void zipEntity(
NetDiskEntity dir,
ZipArchiveOutputStream zipArchiveOutputStream,
NetDiskEntity root,
StopWatch stopWatch) {
if (dir.getEntryType().equals("0")) {
stopWatch.start("创建目录 " + dir.getName() + " " + dir.getId());
zipArchiveOutputStream.putArchiveEntry(new ZipArchiveEntry(getFullName(root, dir)));
zipArchiveOutputStream.closeArchiveEntry();
stopWatch.stop();
} else {
zipArchiveOutputStream.putArchiveEntry(new ZipArchiveEntry(getFullName(root, dir)));
stopWatch.start("下载文件 " + dir.getName() + " " + dir.getId());
OSSObject ossObject =
ossClient.getObject(aliyunProperty.getOSS().getBucketName(), dir.getOssKey());
IOUtils.copy(ossObject.getObjectContent(), zipArchiveOutputStream);
stopWatch.stop();
zipArchiveOutputStream.closeArchiveEntry();
for (NetDiskEntity child : dir.getChildren()) {
zipEntity(child, zipArchiveOutputStream, root, stopWatch);
}
}
}
public NetDiskTreeDto dirTree(IdRequest request) {
StopWatch stopWatch = new StopWatch("查询目录树");
stopWatch.start("查询所有子目录");
List<String> ids = this.repository.findAllSubDir(request.getId());
// NetDiskEntity entity = get(request.getId());
stopWatch.stop();
Integer dirDepth = this.repository.dirDepth(request.getId());
stopWatch.start("查询所有子目录的实体");
List<NetDiskEntity> subDirs = this.repository.findAllById(ids);
StringBuilder sb = new StringBuilder();
stopWatch.stop();
sb.append("select r0 from NetDiskEntity r0 where r0.id = '")
.append(request.getId())
.append("'");
NetDiskEntity rootDir =
subDirs.stream()
.filter(x -> StringUtils.equals(x.getId(), request.getId()))
.findFirst()
.get();
sb.append("\n");
for(int i = 1; i < dirDepth; i++) {
sb.append(" join fetch r").append(i - 1).append(".children as r").append(i).append("\n");
stopWatch.start("构建目录树");
Map<String, List<NetDiskEntity>> parentIdMap =
subDirs.stream()
.collect(
Collectors.groupingBy(
e ->
e.getParent() == null
? "root"
: e.getParent().getId()));
stopWatch.stop();
stopWatch.start("构建目录树2");
for (NetDiskEntity subDir : subDirs) {
subDir.setChildren(parentIdMap.get(subDir.getId()));
}
stopWatch.stop();
stopWatch.start("toTreeDto");
NetDiskTreeDto treeDto = this.mapper.toTreeDto(rootDir);
NetDiskEntity fullRoot = (NetDiskEntity) entityManager.createQuery(sb.toString()).getSingleResult();
stopWatch.stop();
log.info(stopWatch.prettyPrint());
return this.mapper.toTreeDto(fullRoot);
return treeDto;
}
public List<NetDiskDto> idToPath2(IdRequest request) {
String id = request.getId();

View File

@@ -181,6 +181,13 @@ public class SysConfigService extends BaseService<SysConfigEntity, SysConfigRepo
2+"",
0L,
Integer.MAX_VALUE);
addStringConfig(
all,
Constants.SYSCONFIG_REQUEST_SIGN_IGNORE_URLS,
"接口签名忽略URL",
"**/*/downloadBatch,**/*/downloadFileLocal,**/*/downloadFile"
);
}
private void addNumberConfig(
@@ -204,6 +211,26 @@ public class SysConfigService extends BaseService<SysConfigEntity, SysConfigRepo
}
}
private void addStringConfig(
Map<String, SysConfigEntity> all,
String code,
String name,
String value
) {
if (!all.containsKey(code)) {
SysConfigEntity entity = new SysConfigEntity();
entity.setName(name);
entity.setCode(code);
entity.setConfigVal(value);
entity.setDictCode(null);
entity.setMinValue(null);
entity.setMaxValue(null);
entity.setRegexValidator(null);
entity.setType("1");
repository.save(entity);
}
}
private void addDictConfig(
Map<String, SysConfigEntity> all,
String code,
@@ -230,6 +257,9 @@ public class SysConfigService extends BaseService<SysConfigEntity, SysConfigRepo
SysConfigEntity config = this.repository.findByCode(configKey);
if (config == null){
return null;
}
return config.getConfigVal();
}