v1

.gitignore

@@ -10,3 +10,4 @@ save3
 /output/
 /data/
 /.idea/
+*.txt
\ No newline at end of file

src/main/java/com/hgny/protobuf/Main.java

@@ -27,7 +27,7 @@ import java.util.stream.Stream;
 
 public class Main {
   private static List<String> SKIP_KEY = new ArrayList<>(List.of(new String[]{"mdp", "mdptime", "mdkwh", "mdkwhtime"}));
-  public static void main(String[] args) throws IOException, InvocationTargetException, IllegalAccessException {
+  public static void main(String[] args) throws IOException, InvocationTargetException, IllegalAccessException, InterruptedException {
 //    test2();
 //    test5();
 //    test6();
@@ -41,17 +41,26 @@ public class Main {
       if (gz.isDirectory()) {
         continue;
       }
-      Tools.extraTarGz(gz, "./data");
-    }
+      String gzName = gz.getName();
+      System.out.println("parse: " + gzName);
+      String folderName = Tools.extraTarGz(gz, "./data");
       // 遍历目录解压tar.bz2
-    for (File file : new File("./data").listFiles()) {
+      for (File file : new File("./data/" + folderName).listFiles()) {
         if (!file.isDirectory()) {
           continue;
         }
         String name = file.getName();
-      scanDir(file, "./data/" + name, 0);
-    }
+        scanDir(file, "./data/" + folderName + "/" + name, 1);
+      }
+      // 处理完成,删除解压出来的文件夹
+      int index = gzName.indexOf(".");
+      String fName = folderName.substring(0, index);
+      deleteFolder(new File("./data/" + fName));
+      Tools.compressTarGz("./output/" + fName, new File("./output/" + fName + ".tar.gz"));
+      deleteFolder(new File("./output/" + fName));
       System.out.println("spend: " + (System.currentTimeMillis() - start) + "ms");
+      start = System.currentTimeMillis();
+    }
     // 压缩
 //    for (File file : new File("./data").listFiles()) {
 //      if (!file.isDirectory()) {
@@ -190,6 +199,8 @@ public class Main {
   }
 
   private static void deleteFolder(File file) {
+    // 释放stream占用的文件
+    System.gc();
     for (File f : file.listFiles()) {
       if (f.isDirectory()) {
         deleteFolder(f);

src/main/java/com/hgny/protobuf/utils/Tools.java

@@ -4,10 +4,15 @@ import com.google.gson.Gson;
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
 
 import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
@@ -18,6 +23,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 public class Tools {
 
@@ -26,16 +32,16 @@ public class Tools {
     return Integer.parseInt(t.substring(0, t.length() - 3));
   }
 
-  public static void extraTarGz(File file, String path) {
+  public static String extraTarGz(File file, String path) {
     if (!file.getName().contains(".tar.gz")) {
-      return;
+      return null;
     }
     try (TarArchiveInputStream inputStream = new TarArchiveInputStream(
         new GZIPInputStream(
             new FileInputStream(file)
         )
     )) {
-      extraTar(inputStream, path);
+      return extraTar(inputStream, path);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
@@ -70,7 +76,6 @@ public class Tools {
         createDir(path, entryName);
       } else {
         createDir(tmpFile.getParent() + "/", null);
-        System.out.println("extra: " + tmpFile.getPath());
         try (FileOutputStream outputStream = new FileOutputStream(tmpFile)) {
           int read = 0;
           byte[] buffer = new byte[1024];
@@ -120,13 +125,53 @@ public class Tools {
     try (FileOutputStream outputStream = new FileOutputStream(file);
          BZip2CompressorOutputStream zip = new BZip2CompressorOutputStream(outputStream, 9)
     ) {
-      System.out.println("compress: " + file.getPath());
       zip.write(data.getBytes(StandardCharsets.UTF_8));
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  public static void compressTarGz(String path, File file) throws IOException {
+    createTarGZ(path, file);
+  }
+
+  public static void createTarGZ(String path, File file)
+      throws IOException
+  {
+    try (
+        FileOutputStream fOut = new FileOutputStream(file);
+        BufferedOutputStream bOut = new BufferedOutputStream(fOut);
+        GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(bOut);
+        TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)
+        ) {
+      addFileToTarGz(tOut, path, "");
+      tOut.flush();
+    }
+  }
+
+  private static void addFileToTarGz(TarArchiveOutputStream tOut, String path, String base)
+      throws IOException
+  {
+    File f = new File(path);
+    String entryName = base + f.getName();
+    TarArchiveEntry tarEntry = new TarArchiveEntry(f, entryName);
+    tOut.putArchiveEntry(tarEntry);
+
+    if (f.isFile()) {
+      IOUtils.copy(new FileInputStream(f), tOut);
+      tOut.flush();
+      tOut.closeArchiveEntry();
+    } else {
+      tOut.closeArchiveEntry();
+      File[] children = f.listFiles();
+      if (children != null) {
+        for (File child : children) {
+          addFileToTarGz(tOut, child.getAbsolutePath(), entryName + "/");
+        }
+      }
+    }
+  }
+
   public static void createDir(String path, String subPath) {
     File file = new File(path);
     if (!(subPath == null || subPath.trim().equals(""))) {