From dcd6cbba80936c41c006c7524e939781c42c3f0a Mon Sep 17 00:00:00 2001 From: ShadelessFox Date: Sat, 9 Mar 2024 22:04:32 +0100 Subject: [PATCH] Archives: lil refactor; replace listeners with a message bus --- .../com/shade/decima/model/app/Project.java | 13 +- .../decima/model/app/ProjectManager.java | 4 + .../decima/model/app/ProjectManagerImpl.java | 13 +- .../shade/decima/model/archive/Archive.java | 3 + .../decima/model/archive/ArchiveManager.java | 14 + .../shade/decima/model/packfile/Packfile.java | 384 ++++++++++++++++-- .../decima/model/packfile/PackfileBase.java | 352 ---------------- .../decima/model/packfile/PackfileFile.java | 4 +- .../model/packfile/PackfileManager.java | 46 ++- .../decima/model/packfile/PackfileWriter.java | 40 +- .../packfile/prefetch/PrefetchUpdater.java | 7 +- .../packfile/resource/PackfileResource.java | 5 +- .../com/shade/decima/model/util/FilePath.java | 6 +- .../cli/commands/DumpEntryPointNames.java | 4 +- .../decima/cli/commands/DumpFilePaths.java | 14 +- .../cli/commands/DumpFileReferences.java | 8 +- .../decima/cli/commands/RepackArchive.java | 12 +- .../ui/data/editors/ReferenceValueEditor.java | 5 +- .../decima/ui/dialogs/FindFilesDialog.java | 17 +- .../ui/dialogs/PersistChangesDialog.java | 15 +- .../decima/ui/navigator/NavigatorView.java | 22 +- .../navigator/impl/NavigatorPackfileNode.java | 5 +- .../impl/NavigatorPackfilesNode.java | 2 +- .../navigator/impl/NavigatorProjectNode.java | 34 +- .../model/packfile/PackfileWriterTest.java | 4 +- 25 files changed, 517 insertions(+), 516 deletions(-) create mode 100644 modules/decima-model/src/main/java/com/shade/decima/model/archive/ArchiveManager.java delete mode 100644 modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileBase.java diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/app/Project.java b/modules/decima-model/src/main/java/com/shade/decima/model/app/Project.java index 55b65db7d..6f72bd1ca 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/app/Project.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/app/Project.java @@ -4,7 +4,6 @@ import com.shade.decima.model.app.impl.HZDPackfileProvider; import com.shade.decima.model.base.CoreBinary; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.decima.model.packfile.PackfileManager; import com.shade.decima.model.packfile.PackfileProvider; import com.shade.decima.model.packfile.prefetch.PrefetchUpdater; @@ -42,8 +41,8 @@ public class Project implements Closeable { this.container = container; this.typeRegistry = new RTTITypeRegistry(container); this.coreFileReader = new CoreBinary.Reader(typeRegistry); - this.packfileManager = new PackfileManager(); this.oodle = Oodle.acquire(container.getCompressorPath()); + this.packfileManager = new PackfileManager(oodle); mountDefaults(); } @@ -58,13 +57,13 @@ private void mountDefaults() throws IOException { Arrays.stream(packfileProvider.getPackfiles(this)).parallel().forEach(info -> { try { - packfileManager.mount(info, oodle); + packfileManager.mountPackfile(info); } catch (IOException e) { log.error("Can't mount packfile '{}'", info.path(), e); } }); - log.info("Found and mounted {} packfiles in {} ms", packfileManager.getPackfiles().size(), System.currentTimeMillis() - start); + log.info("Found and mounted {} packfiles in {} ms", packfileManager.getArchives().size(), System.currentTimeMillis() - start); } @NotNull @@ -118,7 +117,7 @@ public Map listFileLinks() throws IOException { final long[][] refs = new long[files.length][]; for (int i = 0; i < files.length; i++) { - hashes[i] = PackfileBase.getPathHash(PackfileBase.getNormalizedPath(files[i].str("Path"))); + hashes[i] = Packfile.getPathHash(Packfile.getNormalizedPath(files[i].str("Path"))); } for (int i = 0, j = 0; i < files.length; i++, j++) { @@ -176,8 +175,8 @@ private Stream getPrefetchFiles() throws IOException { final RTTIObject[] files = list.get("Files"); return Stream.concat( - Arrays.stream(files).map(entry -> PackfileBase.getNormalizedPath(entry.str("Path"))), - Arrays.stream(files).map(entry -> PackfileBase.getNormalizedPath(entry.str("Path")) + ".stream") + Arrays.stream(files).map(entry -> Packfile.getNormalizedPath(entry.str("Path"))), + Arrays.stream(files).map(entry -> Packfile.getNormalizedPath(entry.str("Path")) + ".stream") ); } diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManager.java b/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManager.java index ed2f05f3f..f20eb9774 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManager.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManager.java @@ -6,6 +6,7 @@ import com.shade.util.Nullable; import java.io.IOException; +import java.util.Collection; import java.util.UUID; public interface ProjectManager { @@ -28,6 +29,9 @@ static ProjectManager getInstance() { @NotNull ProjectContainer[] getProjects(); + @NotNull + Collection getOpenProjects(); + @NotNull Project openProject(@NotNull ProjectContainer container) throws IOException; diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManagerImpl.java b/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManagerImpl.java index 70b55bf83..045e9b0b2 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManagerImpl.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/app/ProjectManagerImpl.java @@ -10,9 +10,7 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.prefs.Preferences; @Service(ProjectManager.class) @@ -78,6 +76,15 @@ public ProjectContainer[] getProjects() { .toArray(ProjectContainer[]::new); } + @NotNull + @Override + public Collection getOpenProjects() { + return projects.values().stream() + .map(info -> info.project) + .filter(Objects::nonNull) + .toList(); + } + @NotNull @Override public synchronized Project openProject(@NotNull ProjectContainer container) throws IOException { diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/archive/Archive.java b/modules/decima-model/src/main/java/com/shade/decima/model/archive/Archive.java index 079ef4c7f..5f58bf459 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/archive/Archive.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/archive/Archive.java @@ -6,6 +6,9 @@ import java.nio.file.Path; public interface Archive extends Closeable { + @NotNull + ArchiveManager getManager(); + @NotNull String getId(); diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/archive/ArchiveManager.java b/modules/decima-model/src/main/java/com/shade/decima/model/archive/ArchiveManager.java new file mode 100644 index 000000000..9068c4d9c --- /dev/null +++ b/modules/decima-model/src/main/java/com/shade/decima/model/archive/ArchiveManager.java @@ -0,0 +1,14 @@ +package com.shade.decima.model.archive; + +import com.shade.util.NotNull; + +import java.io.Closeable; +import java.util.Collection; + +public interface ArchiveManager extends Closeable { + @NotNull + ArchiveFile getFile(@NotNull String identifier); + + @NotNull + Collection getArchives(); +} diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/Packfile.java b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/Packfile.java index 2e7c15ff2..32a761960 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/Packfile.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/Packfile.java @@ -2,37 +2,54 @@ import com.shade.decima.model.archive.Archive; import com.shade.decima.model.archive.ArchiveFile; +import com.shade.decima.model.archive.ArchiveManager; import com.shade.decima.model.packfile.edit.Change; import com.shade.decima.model.packfile.resource.Resource; import com.shade.decima.model.util.FilePath; import com.shade.decima.model.util.Oodle; +import com.shade.decima.model.util.hash.MurmurHash3; +import com.shade.platform.model.messages.MessageBus; +import com.shade.platform.model.messages.Topic; import com.shade.platform.model.util.BufferUtils; import com.shade.platform.model.util.IOUtils; import com.shade.util.NotNull; import com.shade.util.Nullable; -import javax.swing.event.EventListenerList; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.*; -public class Packfile extends PackfileBase implements Archive, Comparable { - private SeekableByteChannel channel; +public class Packfile implements Archive, Comparable { + public static final Topic CHANGES = Topic.create("packfile", PackfileChangeListener.class); + + public static final long[] HEADER_KEY = {0xF41CAB62FA3A9443L, 0xD2A89E3EF376811CL}; + public static final long[] DATA_KEY = {0x7E159D956C084A37L, 0x18AA7D3F3D5AF7E8L}; + public static final int MAGIC_PLAIN = 0x20304050; + public static final int MAGIC_ENCRYPTED = 0x21304050; + + private final PackfileManager manager; private final Oodle oodle; private final PackfileInfo info; private final Map changes = new HashMap<>(); - private final EventListenerList listeners = new EventListenerList(); - public Packfile(@NotNull Path path, @NotNull Oodle oodle) throws IOException { - this(new PackfileInfo(path, IOUtils.getBasename(path), null), oodle); - } + private Header header; + /** {@link FileEntry#hash()} to {@link FileEntry} mappings. */ + private final SortedMap files = new TreeMap<>(Long::compareUnsigned); + /** {@link Span#offset()} of {@link ChunkEntry#decompressed()} to {@link ChunkEntry} mappings. */ + private final NavigableMap chunks = new TreeMap<>(Long::compareUnsigned); + private SeekableByteChannel channel; - Packfile(@NotNull PackfileInfo info, @NotNull Oodle oodle) throws IOException { + Packfile(@NotNull PackfileManager manager, @NotNull Oodle oodle, @NotNull PackfileInfo info) throws IOException { + this.manager = manager; this.oodle = oodle; this.info = info; @@ -40,6 +57,103 @@ public Packfile(@NotNull Path path, @NotNull Oodle oodle) throws IOException { validate(); } + @NotNull + @Override + public ArchiveManager getManager() { + return manager; + } + + @Nullable + public FileEntry getFileEntry(@NotNull String path) { + return files.get(getPathHash(getNormalizedPath(path))); + } + + @Nullable + public FileEntry getFileEntry(long hash) { + return files.get(hash); + } + + @NotNull + public Collection getFileEntries() { + return files.values(); + } + + @NotNull + public NavigableMap getChunkEntries(@NotNull Span span) { + final NavigableMap map = chunks.subMap( + chunks.floorKey(span.offset()), true, + chunks.floorKey(span.offset() + span.size()), true + ); + + if (map.isEmpty()) { + throw new IllegalArgumentException(String.format("Can't find any chunk entries for span starting at %#x (size: %#x)", span.offset(), span.size())); + } + + assert map.firstEntry().getValue().decompressed().contains(span.offset()); + assert map.lastEntry().getValue().decompressed().contains(span.offset() + span.size()); + + return map; + } + + public boolean contains(long hash) { + return files.containsKey(hash); + } + + @NotNull + public static String getNormalizedPath(@NotNull String path) { + return getNormalizedPath(path, true); + } + + @NotNull + public static String getNormalizedPath(@NotNull String path, boolean normalizeExtension) { + if (path.isEmpty()) { + return path; + } + + path = path.replace("\\", "/"); + + while (!path.isEmpty() && path.charAt(0) == '/') { + path = path.substring(1); + } + + if (normalizeExtension) { + final String extension = IOUtils.getExtension(path); + + if (!extension.equals("core") && !extension.equals("stream")) { + path += ".core"; + } + } + + return path; + } + + public static long getPathHash(@NotNull String path) { + final byte[] data = path.getBytes(StandardCharsets.UTF_8); + final byte[] cstr = Arrays.copyOf(data, data.length + 1); + return MurmurHash3.mmh3(cstr)[0]; + } + + private static void swizzle(@NotNull ByteBuffer target, int key1, int key2) { + final ByteBuffer buffer = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + final ByteBuffer slice = target.slice().order(ByteOrder.LITTLE_ENDIAN); + + buffer.putLong(0, HEADER_KEY[0]); + buffer.putLong(8, HEADER_KEY[1]); + buffer.putInt(0, key1); + + final long[] hash1 = MurmurHash3.mmh3(buffer.array(), 0, 16); + slice.putLong(0, slice.getLong(0) ^ hash1[0]); + slice.putLong(8, slice.getLong(8) ^ hash1[1]); + + buffer.putLong(0, HEADER_KEY[0]); + buffer.putLong(8, HEADER_KEY[1]); + buffer.putInt(0, key2); + + final long[] hash2 = MurmurHash3.mmh3(buffer.array(), 0, 16); + slice.putLong(16, slice.getLong(16) ^ hash2[0]); + slice.putLong(24, slice.getLong(24) ^ hash2[1]); + } + public synchronized void reload(boolean purgeChanges) throws IOException { if (purgeChanges) { clearChanges(); @@ -105,29 +219,21 @@ public Change getChange(long hash) { public void addChange(@NotNull FilePath path, @NotNull Change change) { changes.put(path, change); - - for (PackfileChangeListener listener : listeners.getListeners(PackfileChangeListener.class)) { - listener.fileChanged(this, path, change); - } + MessageBus.getInstance().publisher(CHANGES).fileChanged(this, path, change); } public void removeChange(@NotNull FilePath path) { final Change change = changes.remove(path); - - if (change != null) { - for (PackfileChangeListener listener : listeners.getListeners(PackfileChangeListener.class)) { - listener.fileChanged(this, path, change); - } + if (change == null) { + return; } + MessageBus.getInstance().publisher(CHANGES).fileChanged(this, path, change); } public void clearChanges() { for (Map.Entry change : changes.entrySet()) { - for (PackfileChangeListener listener : listeners.getListeners(PackfileChangeListener.class)) { - listener.fileChanged(this, change.getKey(), change.getValue()); - } + MessageBus.getInstance().publisher(CHANGES).fileChanged(this, change.getKey(), change.getValue()); } - changes.clear(); } @@ -149,14 +255,6 @@ public boolean hasChangesInPath(@NotNull FilePath other) { return false; } - public void addChangeListener(@NotNull PackfileChangeListener listener) { - listeners.add(PackfileChangeListener.class, listener); - } - - public void removeChangeListener(@NotNull PackfileChangeListener listener) { - listeners.remove(PackfileChangeListener.class, listener); - } - @NotNull @Override public String getId() { @@ -232,7 +330,7 @@ public String toString() { return "Packfile[" + info.path() + ']'; } - private void read() throws IOException { + protected void read() throws IOException { channel = Files.newByteChannel(info.path(), StandardOpenOption.READ); header = Header.read(BufferUtils.readFromChannel(channel, Header.BYTES)); @@ -252,7 +350,7 @@ private void read() throws IOException { } } - private void validate() throws IOException { + protected void validate() throws IOException { final long actualHeaderSize = Header.BYTES + header.fileEntryCount() * FileEntry.BYTES + (long) header.chunkEntryCount() * ChunkEntry.BYTES; long actualFileSize = actualHeaderSize; long actualDataSize = 0; @@ -315,6 +413,224 @@ private void ensureOpen() throws IOException { } } + public record Header(int magic, int key, long fileSize, long dataSize, long fileEntryCount, int chunkEntryCount, int chunkEntrySize) { + public static final int BYTES = 40; + + @NotNull + public static Header read(@NotNull ByteBuffer buffer) throws IOException { + assert buffer.remaining() >= BYTES; + + final var magic = buffer.getInt(); + final var key = buffer.getInt(); + + if (magic != MAGIC_PLAIN && magic != MAGIC_ENCRYPTED) { + throw new IOException("File magic is invalid, expected %x or %x, got %d".formatted(MAGIC_PLAIN, MAGIC_ENCRYPTED, magic)); + } + + if (isEncrypted(magic)) { + swizzle(buffer, key, key + 1); + } + + final var fileSize = buffer.getLong(); + final var dataSize = buffer.getLong(); + final var fileEntryCount = buffer.getLong(); + final var chunkEntryCount = buffer.getInt(); + final var chunkEntrySize = buffer.getInt(); + + return new Header(magic, key, fileSize, dataSize, fileEntryCount, chunkEntryCount, chunkEntrySize); + } + + public void write(@NotNull ByteBuffer buffer) { + assert buffer.remaining() >= BYTES; + + buffer.putInt(magic); + buffer.putInt(key); + + final int position = buffer.position(); + + buffer.putLong(fileSize); + buffer.putLong(dataSize); + buffer.putLong(fileEntryCount); + buffer.putInt(chunkEntryCount); + buffer.putInt(chunkEntrySize); + + if (isEncrypted()) { + swizzle(buffer.slice(position, 32), key, key + 1); + } + } + + public static boolean isEncrypted(int magic) { + return magic == MAGIC_ENCRYPTED; + } + + public boolean isEncrypted() { + return isEncrypted(magic); + } + } + + public record FileEntry(int index, int key, long hash, @NotNull Span span) implements Comparable { + public static final int BYTES = 32; + + @NotNull + public static FileEntry read(@NotNull ByteBuffer buffer, boolean encrypted) { + assert buffer.remaining() >= BYTES; + + if (encrypted) { + final int base = buffer.position(); + final int key1 = buffer.getInt(base + 4); + final int key2 = buffer.getInt(base + 28); + + swizzle(buffer, key1, key2); + + buffer.putInt(base + 4, key1); + buffer.putInt(base + 28, key2); + } + + final var index = buffer.getInt(); + final var key = buffer.getInt(); + final var hash = buffer.getLong(); + final var span = Span.read(buffer); + + return new FileEntry(index, key, hash, span); + } + + public void write(@NotNull ByteBuffer buffer, boolean encrypt) { + assert buffer.remaining() >= BYTES; + + final int base = buffer.position(); + + buffer.putInt(index); + buffer.putInt(key); + buffer.putLong(hash); + span.write(buffer); + + if (encrypt) { + swizzle(buffer.slice(base, 32), key, span.key); + + buffer.putInt(base + 4, key); + buffer.putInt(base + 28, span.key); + } + } + + @Override + public int compareTo(FileEntry o) { + return Long.compareUnsigned(hash, o.hash); + } + } + + public record ChunkEntry(@NotNull Span decompressed, @NotNull Span compressed) implements Comparable { + public static final int BYTES = 32; + + private static final ThreadLocal MD5 = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + }); + + public static ChunkEntry read(@NotNull ByteBuffer buffer, boolean encrypted) { + assert buffer.remaining() >= BYTES; + + if (encrypted) { + final int base = buffer.position(); + final int key1 = buffer.getInt(base + 12); + final int key2 = buffer.getInt(base + 28); + + Packfile.swizzle(buffer, key1, key2); + + buffer.putInt(base + 12, key1); + buffer.putInt(base + 28, key2); + } + + final var uncompressed = Span.read(buffer); + final var compressed = Span.read(buffer); + + return new ChunkEntry(uncompressed, compressed); + } + + public void write(@NotNull ByteBuffer buffer, boolean encrypt) { + assert buffer.remaining() >= BYTES; + + final int base = buffer.position(); + + decompressed.write(buffer); + compressed.write(buffer); + + if (encrypt) { + Packfile.swizzle(buffer.slice(base, 32), decompressed.key, compressed.key); + + buffer.putInt(base + 12, decompressed.key); + buffer.putInt(base + 28, compressed.key); + } + } + + public void swizzle(@NotNull ByteBuffer buffer) { + swizzle(buffer, decompressed); + } + + public static void swizzle(@NotNull ByteBuffer target, @NotNull Span decompressed) { + final ByteBuffer buffer = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + buffer.putLong(decompressed.offset); + buffer.putInt(decompressed.size); + buffer.putInt(decompressed.key); + + final long[] hash1 = MurmurHash3.mmh3(buffer.array()); + buffer.putLong(0, hash1[0] ^ DATA_KEY[0]); + buffer.putLong(8, hash1[1] ^ DATA_KEY[1]); + + final byte[] hash2 = MD5.get().digest(buffer.array()); + buffer.put(0, hash2); + + final ByteBuffer slice = target.slice().order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0, limit = slice.limit() & ~7; i < limit; i += 8) { + // Process 8 bytes at a time + slice.putLong(i, slice.getLong(i) ^ buffer.getLong(i & 15)); + } + for (int i = slice.limit() & ~7, limit = slice.limit(); i < limit; i++) { + // Process remaining bytes + slice.put(i, (byte) (slice.get(i) ^ buffer.get(i & 15))); + } + } + + @Override + public int compareTo(ChunkEntry o) { + return Long.compareUnsigned(decompressed.offset, o.decompressed.offset); + } + } + + public record Span(long offset, int size, int key) implements Comparable { + public static final int BYTES = 16; + + @NotNull + public static Span read(@NotNull ByteBuffer buffer) { + assert buffer.remaining() >= BYTES; + + final var offset = buffer.getLong(); + final var size = buffer.getInt(); + final var key = buffer.getInt(); + + return new Span(offset, size, key); + } + + public void write(@NotNull ByteBuffer buffer) { + assert buffer.remaining() >= BYTES; + + buffer.putLong(offset); + buffer.putInt(size); + buffer.putInt(key); + } + + public boolean contains(long offset) { + return offset >= this.offset && offset <= this.offset + size; + } + + @Override + public int compareTo(@NotNull Span other) { + return Long.compare(offset, other.offset); + } + } + private static class ResourceInputStream extends InputStream { private final Resource resource; @@ -325,7 +641,7 @@ public ResourceInputStream(@NotNull Resource resource) { @Override public int read() throws IOException { final byte[] buffer = new byte[1]; - final int length = (int) resource.read(ByteBuffer.wrap(buffer)); + final int length = resource.read(ByteBuffer.wrap(buffer)); if (length <= 0) { return -1; @@ -336,13 +652,13 @@ public int read() throws IOException { @Override public int read(@NotNull byte[] b, int off, int len) throws IOException { - return (int) resource.read(ByteBuffer.wrap(b, off, len)); + return resource.read(ByteBuffer.wrap(b, off, len)); } @Override public byte[] readAllBytes() throws IOException { final byte[] buffer = new byte[resource.size()]; - final int length = (int) resource.read(ByteBuffer.wrap(buffer)); + final int length = resource.read(ByteBuffer.wrap(buffer)); return Arrays.copyOf(buffer, length); } } diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileBase.java b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileBase.java deleted file mode 100644 index c649c7e57..000000000 --- a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileBase.java +++ /dev/null @@ -1,352 +0,0 @@ -package com.shade.decima.model.packfile; - -import com.shade.decima.model.util.hash.MurmurHash3; -import com.shade.platform.model.util.IOUtils; -import com.shade.util.NotNull; -import com.shade.util.Nullable; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.*; - -public abstract class PackfileBase { - public static final long[] HEADER_KEY = {0xF41CAB62FA3A9443L, 0xD2A89E3EF376811CL}; - public static final long[] DATA_KEY = {0x7E159D956C084A37L, 0x18AA7D3F3D5AF7E8L}; - - public static final int MAGIC_PLAIN = 0x20304050; - public static final int MAGIC_ENCRYPTED = 0x21304050; - - protected Header header; - - /** - * {@link FileEntry#hash()} to {@link FileEntry} mappings. - */ - protected final SortedMap files; - - /** - * {@link Span#offset()} of {@link ChunkEntry#decompressed()} to {@link ChunkEntry} mappings. - */ - protected final NavigableMap chunks; - - protected PackfileBase() { - this.files = new TreeMap<>(Long::compareUnsigned); - this.chunks = new TreeMap<>(Long::compareUnsigned); - } - - @Nullable - public FileEntry getFileEntry(@NotNull String path) { - return files.get(getPathHash(getNormalizedPath(path))); - } - - @Nullable - public FileEntry getFileEntry(long hash) { - return files.get(hash); - } - - @NotNull - public Collection getFileEntries() { - return files.values(); - } - - @NotNull - public NavigableMap getChunkEntries(@NotNull Span span) { - final NavigableMap map = chunks.subMap( - chunks.floorKey(span.offset()), true, - chunks.floorKey(span.offset() + span.size()), true - ); - - if (map.isEmpty()) { - throw new IllegalArgumentException(String.format("Can't find any chunk entries for span starting at %#x (size: %#x)", span.offset(), span.size())); - } - - assert map.firstEntry().getValue().decompressed().contains(span.offset()); - assert map.lastEntry().getValue().decompressed().contains(span.offset() + span.size()); - - return map; - } - - public boolean contains(long hash) { - return files.containsKey(hash); - } - - public boolean isEmpty() { - return header.fileEntryCount() == 0; - } - - @NotNull - public static String getNormalizedPath(@NotNull String path) { - return getNormalizedPath(path, true); - } - - @NotNull - public static String getNormalizedPath(@NotNull String path, boolean normalizeExtension) { - if (path.isEmpty()) { - return path; - } - - path = path.replace("\\", "/"); - - while (!path.isEmpty() && path.charAt(0) == '/') { - path = path.substring(1); - } - - if (normalizeExtension) { - final String extension = IOUtils.getExtension(path); - - if (!extension.equals("core") && !extension.equals("stream")) { - path += ".core"; - } - } - - return path; - } - - public static long getPathHash(@NotNull String path) { - final byte[] data = path.getBytes(StandardCharsets.UTF_8); - final byte[] cstr = Arrays.copyOf(data, data.length + 1); - return MurmurHash3.mmh3(cstr)[0]; - } - - private static void swizzle(@NotNull ByteBuffer target, int key1, int key2) { - final ByteBuffer buffer = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); - final ByteBuffer slice = target.slice().order(ByteOrder.LITTLE_ENDIAN); - - buffer.putLong(0, HEADER_KEY[0]); - buffer.putLong(8, HEADER_KEY[1]); - buffer.putInt(0, key1); - - final long[] hash1 = MurmurHash3.mmh3(buffer.array(), 0, 16); - slice.putLong(0, slice.getLong(0) ^ hash1[0]); - slice.putLong(8, slice.getLong(8) ^ hash1[1]); - - buffer.putLong(0, HEADER_KEY[0]); - buffer.putLong(8, HEADER_KEY[1]); - buffer.putInt(0, key2); - - final long[] hash2 = MurmurHash3.mmh3(buffer.array(), 0, 16); - slice.putLong(16, slice.getLong(16) ^ hash2[0]); - slice.putLong(24, slice.getLong(24) ^ hash2[1]); - } - - public record Header(int magic, int key, long fileSize, long dataSize, long fileEntryCount, int chunkEntryCount, int chunkEntrySize) { - public static final int BYTES = 40; - - @NotNull - public static Header read(@NotNull ByteBuffer buffer) throws IOException { - assert buffer.remaining() >= BYTES; - - final var magic = buffer.getInt(); - final var key = buffer.getInt(); - - if (magic != MAGIC_PLAIN && magic != MAGIC_ENCRYPTED) { - throw new IOException("File magic is invalid, expected %x or %x, got %d".formatted(MAGIC_PLAIN, MAGIC_ENCRYPTED, magic)); - } - - if (isEncrypted(magic)) { - swizzle(buffer, key, key + 1); - } - - final var fileSize = buffer.getLong(); - final var dataSize = buffer.getLong(); - final var fileEntryCount = buffer.getLong(); - final var chunkEntryCount = buffer.getInt(); - final var chunkEntrySize = buffer.getInt(); - - return new Header(magic, key, fileSize, dataSize, fileEntryCount, chunkEntryCount, chunkEntrySize); - } - - public void write(@NotNull ByteBuffer buffer) { - assert buffer.remaining() >= BYTES; - - buffer.putInt(magic); - buffer.putInt(key); - - final int position = buffer.position(); - - buffer.putLong(fileSize); - buffer.putLong(dataSize); - buffer.putLong(fileEntryCount); - buffer.putInt(chunkEntryCount); - buffer.putInt(chunkEntrySize); - - if (isEncrypted()) { - swizzle(buffer.slice(position, 32), key, key + 1); - } - } - - public static boolean isEncrypted(int magic) { - return magic == MAGIC_ENCRYPTED; - } - - public boolean isEncrypted() { - return isEncrypted(magic); - } - } - - public record FileEntry(int index, int key, long hash, @NotNull Span span) implements Comparable { - public static final int BYTES = 32; - - @NotNull - public static FileEntry read(@NotNull ByteBuffer buffer, boolean encrypted) { - assert buffer.remaining() >= BYTES; - - if (encrypted) { - final int base = buffer.position(); - final int key1 = buffer.getInt(base + 4); - final int key2 = buffer.getInt(base + 28); - - swizzle(buffer, key1, key2); - - buffer.putInt(base + 4, key1); - buffer.putInt(base + 28, key2); - } - - final var index = buffer.getInt(); - final var key = buffer.getInt(); - final var hash = buffer.getLong(); - final var span = Span.read(buffer); - - return new FileEntry(index, key, hash, span); - } - - public void write(@NotNull ByteBuffer buffer, boolean encrypt) { - assert buffer.remaining() >= BYTES; - - final int base = buffer.position(); - - buffer.putInt(index); - buffer.putInt(key); - buffer.putLong(hash); - span.write(buffer); - - if (encrypt) { - swizzle(buffer.slice(base, 32), key, span.key); - - buffer.putInt(base + 4, key); - buffer.putInt(base + 28, span.key); - } - } - - @Override - public int compareTo(FileEntry o) { - return Long.compareUnsigned(hash, o.hash); - } - } - - public record ChunkEntry(@NotNull Span decompressed, @NotNull Span compressed) implements Comparable { - public static final int BYTES = 32; - - private static final ThreadLocal MD5 = ThreadLocal.withInitial(() -> { - try { - return MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - }); - - public static ChunkEntry read(@NotNull ByteBuffer buffer, boolean encrypted) { - assert buffer.remaining() >= BYTES; - - if (encrypted) { - final int base = buffer.position(); - final int key1 = buffer.getInt(base + 12); - final int key2 = buffer.getInt(base + 28); - - PackfileBase.swizzle(buffer, key1, key2); - - buffer.putInt(base + 12, key1); - buffer.putInt(base + 28, key2); - } - - final var uncompressed = Span.read(buffer); - final var compressed = Span.read(buffer); - - return new ChunkEntry(uncompressed, compressed); - } - - public void write(@NotNull ByteBuffer buffer, boolean encrypt) { - assert buffer.remaining() >= BYTES; - - final int base = buffer.position(); - - decompressed.write(buffer); - compressed.write(buffer); - - if (encrypt) { - PackfileBase.swizzle(buffer.slice(base, 32), decompressed.key, compressed.key); - - buffer.putInt(base + 12, decompressed.key); - buffer.putInt(base + 28, compressed.key); - } - } - - public void swizzle(@NotNull ByteBuffer buffer) { - swizzle(buffer, decompressed); - } - - public static void swizzle(@NotNull ByteBuffer target, @NotNull Span decompressed) { - final ByteBuffer buffer = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); - buffer.putLong(decompressed.offset); - buffer.putInt(decompressed.size); - buffer.putInt(decompressed.key); - - final long[] hash1 = MurmurHash3.mmh3(buffer.array()); - buffer.putLong(0, hash1[0] ^ DATA_KEY[0]); - buffer.putLong(8, hash1[1] ^ DATA_KEY[1]); - - final byte[] hash2 = MD5.get().digest(buffer.array()); - buffer.put(0, hash2); - - final ByteBuffer slice = target.slice().order(ByteOrder.LITTLE_ENDIAN); - for (int i = 0, limit = slice.limit() & ~7; i < limit; i += 8) { - // Process 8 bytes at a time - slice.putLong(i, slice.getLong(i) ^ buffer.getLong(i & 15)); - } - for (int i = slice.limit() & ~7, limit = slice.limit(); i < limit; i++) { - // Process remaining bytes - slice.put(i, (byte) (slice.get(i) ^ buffer.get(i & 15))); - } - } - - @Override - public int compareTo(ChunkEntry o) { - return Long.compareUnsigned(decompressed.offset, o.decompressed.offset); - } - } - - public record Span(long offset, int size, int key) implements Comparable { - public static final int BYTES = 16; - - @NotNull - public static Span read(@NotNull ByteBuffer buffer) { - assert buffer.remaining() >= BYTES; - - final var offset = buffer.getLong(); - final var size = buffer.getInt(); - final var key = buffer.getInt(); - - return new Span(offset, size, key); - } - - public void write(@NotNull ByteBuffer buffer) { - assert buffer.remaining() >= BYTES; - - buffer.putLong(offset); - buffer.putInt(size); - buffer.putInt(key); - } - - public boolean contains(long offset) { - return offset >= this.offset && offset <= this.offset + size; - } - - @Override - public int compareTo(@NotNull Span other) { - return Long.compare(offset, other.offset); - } - } -} diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileFile.java b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileFile.java index 6dd38de2d..04ef14412 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileFile.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileFile.java @@ -9,9 +9,9 @@ public class PackfileFile implements ArchiveFile { private final Packfile packfile; - private final PackfileBase.FileEntry entry; + private final Packfile.FileEntry entry; - public PackfileFile(@NotNull Packfile packfile, @NotNull PackfileBase.FileEntry entry) { + public PackfileFile(@NotNull Packfile packfile, @NotNull Packfile.FileEntry entry) { this.packfile = packfile; this.entry = entry; } diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileManager.java b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileManager.java index 736193789..8002debc0 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileManager.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileManager.java @@ -1,35 +1,39 @@ package com.shade.decima.model.packfile; +import com.shade.decima.model.archive.ArchiveFile; +import com.shade.decima.model.archive.ArchiveManager; import com.shade.decima.model.packfile.edit.Change; import com.shade.decima.model.util.FilePath; import com.shade.decima.model.util.Oodle; +import com.shade.platform.model.util.IOUtils; import com.shade.util.NotNull; import com.shade.util.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.util.*; -import static com.shade.decima.model.packfile.PackfileBase.*; +import static com.shade.decima.model.packfile.Packfile.*; -public class PackfileManager implements Closeable { +public class PackfileManager implements ArchiveManager { private static final Logger log = LoggerFactory.getLogger(PackfileManager.class); - private final NavigableSet packfiles; + private final NavigableSet packfiles = new TreeSet<>(); + private final Oodle oodle; - public PackfileManager() { - this.packfiles = new TreeSet<>(); + public PackfileManager(@NotNull Oodle oodle) { + this.oodle = oodle; } - public void mount(@NotNull PackfileInfo info, @NotNull Oodle oodle) throws IOException { + public void mountPackfile(@NotNull PackfileInfo info) throws IOException { if (Files.notExists(info.path())) { return; } - final Packfile packfile = new Packfile(info, oodle); + final Packfile packfile = new Packfile(this, oodle, info); synchronized (this) { if (!packfiles.add(packfile)) { @@ -41,6 +45,27 @@ public void mount(@NotNull PackfileInfo info, @NotNull Oodle oodle) throws IOExc log.info("Mounted '{}'", info.path()); } + @NotNull + public Packfile openPackfile(@NotNull Path path) throws IOException { + return new Packfile(this, oodle, new PackfileInfo(path, IOUtils.getBasename(path), null)); + } + + @NotNull + @Override + public ArchiveFile getFile(@NotNull String identifier) { + final Packfile archive = findFirst(identifier); + if (archive == null) { + throw new IllegalArgumentException("Can't find file '%s'".formatted(identifier)); + } + return archive.getFile(identifier); + } + + @NotNull + @Override + public Collection getArchives() { + return packfiles; + } + @Nullable public Packfile findFirst(@NotNull String path) { return findFirst(getPathHash(getNormalizedPath(path))); @@ -95,11 +120,6 @@ public boolean canMergeChanges() { return true; } - @NotNull - public Collection getPackfiles() { - return packfiles; - } - @Override public void close() throws IOException { for (Packfile packfile : packfiles) { diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileWriter.java b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileWriter.java index c2a22256a..853da6534 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileWriter.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/PackfileWriter.java @@ -32,8 +32,8 @@ public long write( @NotNull Options options ) throws IOException { final RandomGenerator random = new SecureRandom(); - final Set files = new TreeSet<>(); - final Set chunks = new TreeSet<>(); + final Set files = new TreeSet<>(); + final Set chunks = new TreeSet<>(); try (ProgressMonitor.Task task = monitor.begin("Write packfile", 2)) { channel.position(computeHeaderSize()); @@ -45,13 +45,13 @@ public long write( } @NotNull - private PackfileBase.Header writeHeader( + private Packfile.Header writeHeader( @NotNull ProgressMonitor monitor, @NotNull SeekableByteChannel channel, @NotNull RandomGenerator random, @NotNull Options options, - @NotNull Set files, - @NotNull Set chunks + @NotNull Set files, + @NotNull Set chunks ) throws IOException { final long decompressedSize = chunks.stream() .mapToLong(entry -> entry.decompressed().size()) @@ -67,8 +67,8 @@ private PackfileBase.Header writeHeader( .allocate(headerSize) .order(ByteOrder.LITTLE_ENDIAN); - final PackfileBase.Header header = new PackfileBase.Header( - options.encrypt() ? PackfileBase.MAGIC_ENCRYPTED : PackfileBase.MAGIC_PLAIN, + final Packfile.Header header = new Packfile.Header( + options.encrypt() ? Packfile.MAGIC_ENCRYPTED : Packfile.MAGIC_PLAIN, options.encrypt() ? random.nextInt() : 0, compressedSize + headerSize, decompressedSize, @@ -80,11 +80,11 @@ private PackfileBase.Header writeHeader( try (ProgressMonitor.Task task = monitor.begin("Write header", 1)) { header.write(buffer); - for (PackfileBase.FileEntry file : files) { + for (Packfile.FileEntry file : files) { file.write(buffer, options.encrypt()); } - for (PackfileBase.ChunkEntry chunk : chunks) { + for (Packfile.ChunkEntry chunk : chunks) { chunk.write(buffer, options.encrypt()); } @@ -101,8 +101,8 @@ private void writeData( @NotNull Oodle oodle, @NotNull RandomGenerator random, @NotNull Options options, - @NotNull Set files, - @NotNull Set chunks + @NotNull Set files, + @NotNull Set chunks ) throws IOException { final Queue pending = new ArrayDeque<>(resources); final ByteBuffer decompressed = ByteBuffer.allocate(Oodle.BLOCK_SIZE_BYTES); @@ -124,11 +124,11 @@ private void writeData( if (length <= 0) { pending.remove().close(); - files.add(new PackfileBase.FileEntry( + files.add(new Packfile.FileEntry( files.size(), options.encrypt() ? random.nextInt() : 0, resource.hash(), - new PackfileBase.Span( + new Packfile.Span( fileDataOffset, resource.size(), options.encrypt() ? random.nextInt() : 0 @@ -153,23 +153,23 @@ private void writeData( final ByteBuffer compressed = oodle.compress(decompressed.slice(), options.compression()); - final PackfileBase.Span decompressedSpan = new PackfileBase.Span( + final Packfile.Span decompressedSpan = new Packfile.Span( chunkDataDecompressedOffset, decompressed.remaining(), options.encrypt() ? random.nextInt() : 0 ); - final PackfileBase.Span compressedSpan = new PackfileBase.Span( + final Packfile.Span compressedSpan = new Packfile.Span( chunkDataCompressedOffset, compressed.remaining(), options.encrypt() ? random.nextInt() : 0 ); if (options.encrypt()) { - PackfileBase.ChunkEntry.swizzle(compressed, decompressedSpan); + Packfile.ChunkEntry.swizzle(compressed, decompressedSpan); } - chunks.add(new PackfileBase.ChunkEntry(decompressedSpan, compressedSpan)); + chunks.add(new Packfile.ChunkEntry(decompressedSpan, compressedSpan)); chunkDataDecompressedOffset += decompressed.remaining(); chunkDataCompressedOffset += compressed.remaining(); @@ -193,9 +193,9 @@ public void close() throws IOException { } private int computeHeaderSize() { - return PackfileBase.Header.BYTES - + PackfileBase.FileEntry.BYTES * resources.size() - + PackfileBase.ChunkEntry.BYTES * computeChunksCount(); + return Packfile.Header.BYTES + + Packfile.FileEntry.BYTES * resources.size() + + Packfile.ChunkEntry.BYTES * computeChunksCount(); } private int computeChunksCount() { diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/prefetch/PrefetchUpdater.java b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/prefetch/PrefetchUpdater.java index 0e6221e63..7731dc1cd 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/prefetch/PrefetchUpdater.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/prefetch/PrefetchUpdater.java @@ -3,7 +3,6 @@ import com.shade.decima.model.app.Project; import com.shade.decima.model.archive.ArchiveFile; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.decima.model.packfile.PackfileManager; import com.shade.decima.model.packfile.edit.Change; import com.shade.decima.model.packfile.edit.MemoryChange; @@ -96,7 +95,7 @@ private static void rebuildPrefetch( for (String path : prefetch.files()) { final Resource resource; try { - resource = fileSupplier.get(PackfileBase.getPathHash(PackfileBase.getNormalizedPath(path))); + resource = fileSupplier.get(Packfile.getPathHash(Packfile.getNormalizedPath(path))); } catch (IOException e) { log.error("Unable to get resource for '{}': {}", path, e.getMessage()); continue; @@ -204,7 +203,7 @@ static FileSupplier ofAll(@NotNull PackfileManager packfileManager) { log.error("Can't find packfile for hash {}", hash); return null; } - final PackfileBase.FileEntry entry = packfile.getFileEntry(hash); + final Packfile.FileEntry entry = packfile.getFileEntry(hash); if (entry == null) { log.error("Can't find file entry for hash {}", hash); return null; @@ -218,7 +217,7 @@ static FileSupplier ofAll(@NotNull PackfileManager packfileManager) { */ @NotNull static FileSupplier ofChanged(@NotNull PackfileManager packfileManager) { - final Map files = packfileManager.getPackfiles().stream() + final Map files = packfileManager.getArchives().stream() .filter(Packfile::hasChanges) .flatMap(packfile -> packfile.getChanges().entrySet().stream()) .collect(Collectors.toMap( diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/resource/PackfileResource.java b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/resource/PackfileResource.java index 1e8bce348..64eb71454 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/packfile/resource/PackfileResource.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/packfile/resource/PackfileResource.java @@ -1,7 +1,6 @@ package com.shade.decima.model.packfile.resource; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.util.NotNull; import java.io.IOException; @@ -12,12 +11,12 @@ public class PackfileResource implements Resource { private static final int DEFAULT_BUFFER_SIZE = 8192; private final Packfile packfile; - private final PackfileBase.FileEntry entry; + private final Packfile.FileEntry entry; private final byte[] buffer; private InputStream stream; - public PackfileResource(@NotNull Packfile packfile, @NotNull PackfileBase.FileEntry entry) { + public PackfileResource(@NotNull Packfile packfile, @NotNull Packfile.FileEntry entry) { this.packfile = packfile; this.entry = entry; this.buffer = new byte[DEFAULT_BUFFER_SIZE]; diff --git a/modules/decima-model/src/main/java/com/shade/decima/model/util/FilePath.java b/modules/decima-model/src/main/java/com/shade/decima/model/util/FilePath.java index 255b4037a..2d393d322 100644 --- a/modules/decima-model/src/main/java/com/shade/decima/model/util/FilePath.java +++ b/modules/decima-model/src/main/java/com/shade/decima/model/util/FilePath.java @@ -1,6 +1,6 @@ package com.shade.decima.model.util; -import com.shade.decima.model.packfile.PackfileBase; +import com.shade.decima.model.packfile.Packfile; import com.shade.platform.model.util.IOUtils; import com.shade.util.NotNull; @@ -21,11 +21,11 @@ public static FilePath of(@NotNull String path) { @NotNull public static FilePath of(@NotNull String path, boolean computeHash) { - final String normalized = PackfileBase.getNormalizedPath(path); + final String normalized = Packfile.getNormalizedPath(path); final String[] parts = normalized.split("/"); if (computeHash) { - return new FilePath(parts, PackfileBase.getPathHash(normalized)); + return new FilePath(parts, Packfile.getPathHash(normalized)); } else { return new FilePath(parts); } diff --git a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpEntryPointNames.java b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpEntryPointNames.java index 1ed816bc1..81ef22471 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpEntryPointNames.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpEntryPointNames.java @@ -36,11 +36,11 @@ public void run() { final var registry = project.getTypeRegistry(); final var index = new AtomicInteger(); - final var total = manager.getPackfiles().stream() + final var total = manager.getArchives().stream() .mapToInt(packfile -> packfile.getFileEntries().size()) .sum(); - final List names = manager.getPackfiles().parallelStream() + final List names = manager.getArchives().parallelStream() .flatMap(packfile -> packfile.getFileEntries().parallelStream() .flatMap(file -> { try { diff --git a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFilePaths.java b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFilePaths.java index a3c292c80..62c6ff7d5 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFilePaths.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFilePaths.java @@ -1,7 +1,7 @@ package com.shade.decima.cli.commands; import com.shade.decima.model.app.Project; -import com.shade.decima.model.packfile.PackfileBase; +import com.shade.decima.model.packfile.Packfile; import com.shade.decima.model.rtti.types.RTTITypeEnum; import com.shade.util.NotNull; import org.slf4j.Logger; @@ -38,10 +38,10 @@ public void run() { final var manager = project.getPackfileManager(); final var registry = project.getTypeRegistry(); - final var entries = manager.getPackfiles().stream() - .map(PackfileBase::getFileEntries) + final var entries = manager.getArchives().stream() + .map(Packfile::getFileEntries) .flatMap(Collection::stream) - .map(PackfileBase.FileEntry::hash) + .map(Packfile.FileEntry::hash) .collect(Collectors.toSet()); final var languages = Arrays.stream(((RTTITypeEnum) registry.find("ELanguage")).values()) @@ -50,14 +50,14 @@ public void run() { .distinct() .toArray(String[]::new); - final int total = manager.getPackfiles().stream() + final int total = manager.getArchives().stream() .mapToInt(packfile -> packfile.getFileEntries().size()) .sum(); final AtomicInteger index = new AtomicInteger(); log.info("Files found: {} (unique files: {})", total, entries.size()); - final Set paths = manager.getPackfiles().parallelStream() + final Set paths = manager.getArchives().parallelStream() .flatMap(packfile -> packfile.getFileEntries().parallelStream() .flatMap(file -> { try { @@ -89,7 +89,7 @@ public void run() { Arrays.stream(languages).map(lang -> path + ".wem." + lang + ".core.stream") ) )) - .filter(path -> entries.contains(PackfileBase.getPathHash(path))) + .filter(path -> entries.contains(Packfile.getPathHash(path))) ) .collect(TreeSet::new, Set::add, Set::addAll); diff --git a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFileReferences.java b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFileReferences.java index f2e329c7e..dc6be8588 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFileReferences.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/DumpFileReferences.java @@ -1,7 +1,7 @@ package com.shade.decima.cli.commands; import com.shade.decima.model.app.Project; -import com.shade.decima.model.packfile.PackfileBase; +import com.shade.decima.model.packfile.Packfile; import com.shade.decima.model.rtti.RTTIUtils; import com.shade.decima.model.rtti.objects.RTTIReference; import org.slf4j.Logger; @@ -35,11 +35,11 @@ public void run() { final var manager = project.getPackfileManager(); final var index = new AtomicInteger(); - final var total = manager.getPackfiles().stream() + final var total = manager.getArchives().stream() .mapToInt(packfile -> packfile.getFileEntries().size()) .sum(); - final List names = manager.getPackfiles().parallelStream() + final List names = manager.getArchives().parallelStream() .flatMap(packfile -> packfile.getFileEntries().parallelStream() .flatMap(file -> { try { @@ -54,7 +54,7 @@ public void run() { result.add("%#018x,%s,%s".formatted( file.hash(), - PackfileBase.getNormalizedPath(ref.path()), + Packfile.getNormalizedPath(ref.path()), RTTIUtils.uuidToString(ref.uuid()) )); }); diff --git a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/RepackArchive.java b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/RepackArchive.java index 5a303a489..89f6a88f1 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/RepackArchive.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/cli/commands/RepackArchive.java @@ -2,7 +2,6 @@ import com.shade.decima.model.app.Project; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.decima.model.packfile.PackfileWriter; import com.shade.decima.model.packfile.edit.Change; import com.shade.decima.model.packfile.edit.FileChange; @@ -66,13 +65,12 @@ public class RepackArchive implements Callable { @Override public Void call() throws Exception { - final Oodle oodle = project.getCompressor(); final Packfile source; if (truncate) { source = null; } else if (Files.exists(path)) { - source = new Packfile(path, oodle); + source = project.getPackfileManager().openPackfile(path); } else { log.warn("The specified archive file does not exist: " + path); source = null; @@ -112,7 +110,7 @@ public Void call() throws Exception { } if (source != null) { - for (PackfileBase.FileEntry entry : source.getFileEntries()) { + for (Packfile.FileEntry entry : source.getFileEntries()) { if (changes.containsKey(entry.hash())) { continue; } @@ -136,7 +134,7 @@ public Void call() throws Exception { try (FileChannel channel = FileChannel.open(result, WRITE, CREATE, TRUNCATE_EXISTING)) { log.info("Writing data to {}", result.toAbsolutePath()); // TODO: Use console progress monitor here!!! - writer.write(new VoidProgressMonitor(), channel, oodle, new PackfileWriter.Options(compression, encrypt)); + writer.write(new VoidProgressMonitor(), channel, project.getCompressor(), new PackfileWriter.Options(compression, encrypt)); } Files.move(result, path, REPLACE_EXISTING); @@ -154,8 +152,8 @@ private static Map collectChanges(@NotNull Path root) throws IOExc Files.walkFileTree(root, new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - final String path = PackfileBase.getNormalizedPath(root.relativize(file).toString()); - final long hash = PackfileBase.getPathHash(path); + final String path = Packfile.getNormalizedPath(root.relativize(file).toString()); + final long hash = Packfile.getPathHash(path); log.info("Found {}", path); changes.put(hash, new FileChange(file, hash)); diff --git a/modules/decima-ui/src/main/java/com/shade/decima/ui/data/editors/ReferenceValueEditor.java b/modules/decima-ui/src/main/java/com/shade/decima/ui/data/editors/ReferenceValueEditor.java index 9c0733239..d2b91800d 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/ui/data/editors/ReferenceValueEditor.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/ui/data/editors/ReferenceValueEditor.java @@ -3,7 +3,6 @@ import com.formdev.flatlaf.FlatClientProperties; import com.shade.decima.model.app.Project; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.decima.model.rtti.*; import com.shade.decima.model.rtti.objects.RTTIObject; import com.shade.decima.model.rtti.objects.RTTIReference; @@ -139,7 +138,7 @@ private String getPath() { if (path.isEmpty()) { return getCurrentPath(); } else { - return PackfileBase.getNormalizedPath(path); + return Packfile.getNormalizedPath(path); } } @@ -147,7 +146,7 @@ private String getPath() { private String getCurrentPath() { final NodeEditorInput input = (NodeEditorInput) controller.getEditor().getInput(); final String path = input.getNode().getPath().full(); - return PackfileBase.getNormalizedPath(path); + return Packfile.getNormalizedPath(path); } private static class PathPickerDialog extends BaseEditDialog { diff --git a/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/FindFilesDialog.java b/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/FindFilesDialog.java index dc5badc78..1af615030 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/FindFilesDialog.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/FindFilesDialog.java @@ -4,7 +4,6 @@ import com.formdev.flatlaf.icons.FlatSearchWithHistoryIcon; import com.shade.decima.model.app.Project; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.decima.ui.editor.NodeEditorInputLazy; import com.shade.platform.model.runtime.ProgressMonitor; import com.shade.platform.model.util.IOUtils; @@ -228,7 +227,7 @@ public void actionPerformed(ActionEvent event) { private void refreshResults() { ((FilterableTableModel) resultsTable.getModel()).refresh( - PackfileBase.getNormalizedPath(inputField.getText(), false), + Packfile.getNormalizedPath(inputField.getText(), false), strategyCombo.getItemAt(strategyCombo.getSelectedIndex()) ); resultsTable.changeSelection(0, 0, false, false); @@ -258,17 +257,17 @@ private static FileInfoIndex buildFileInfoIndex(@NotNull ProgressMonitor monitor try (var ignored = task.split(1).begin("Add named entries")) { final Map> packfiles = new HashMap<>(); - for (Packfile packfile : project.getPackfileManager().getPackfiles()) { - for (PackfileBase.FileEntry fileEntry : packfile.getFileEntries()) { + for (Packfile packfile : project.getPackfileManager().getArchives()) { + for (Packfile.FileEntry fileEntry : packfile.getFileEntries()) { packfiles.computeIfAbsent(fileEntry.hash(), x -> new ArrayList<>()).add(packfile); } } try (Stream files = project.listAllFiles()) { files.forEach(path -> { - final long hash = PackfileBase.getPathHash(path); + final long hash = Packfile.getPathHash(path); for (Packfile packfile : packfiles.getOrDefault(hash, Collections.emptyList())) { - final PackfileBase.FileEntry entry = Objects.requireNonNull(packfile.getFileEntry(hash)); + final Packfile.FileEntry entry = Objects.requireNonNull(packfile.getFileEntry(hash)); info.add(new FileInfo(packfile, path, hash, entry.span().size())); seen.computeIfAbsent(packfile, x -> new HashSet<>()) .add(hash); @@ -278,12 +277,12 @@ private static FileInfoIndex buildFileInfoIndex(@NotNull ProgressMonitor monitor } try (var ignored = task.split(1).begin("Add unnamed entries")) { - for (Packfile packfile : project.getPackfileManager().getPackfiles()) { + for (Packfile packfile : project.getPackfileManager().getArchives()) { final Set files = seen.get(packfile); if (files == null) { continue; } - for (PackfileBase.FileEntry entry : packfile.getFileEntries()) { + for (Packfile.FileEntry entry : packfile.getFileEntries()) { final long hash = entry.hash(); if (files.contains(hash)) { continue; @@ -368,7 +367,7 @@ public void refresh(@NotNull String query, @NotNull Strategy strategy) { if (matcher.matches()) { hash = Long.parseUnsignedLong(matcher.group(1), 16); } else { - hash = PackfileBase.getPathHash(PackfileBase.getNormalizedPath(query, false)); + hash = Packfile.getPathHash(Packfile.getNormalizedPath(query, false)); } final FileInfo[] output = switch (strategy) { diff --git a/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/PersistChangesDialog.java b/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/PersistChangesDialog.java index d57c8279e..0e133d38d 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/PersistChangesDialog.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/ui/dialogs/PersistChangesDialog.java @@ -3,7 +3,6 @@ import com.shade.decima.model.app.Project; import com.shade.decima.model.base.GameType; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.decima.model.packfile.PackfileWriter; import com.shade.decima.model.packfile.PackfileWriter.Options; import com.shade.decima.model.packfile.edit.Change; @@ -306,7 +305,7 @@ private NavigatorTree createFilteredTree() { } else if (node instanceof NavigatorFileNode n) { return n.getPackfile().hasChangesInPath(n.getPath()); } else if (node instanceof NavigatorPackfilesNode n) { - return Arrays.stream(n.getPackfiles()).anyMatch(Packfile::hasChanges); + return Arrays.stream(n.getArchives()).anyMatch(Packfile::hasChanges); } else { return false; } @@ -329,17 +328,17 @@ private void collectSinglePackfile( boolean append, boolean backup ) throws IOException { + final Project project = root.getProject(); final Packfile packfile; if (append && Files.exists(path)) { - packfile = new Packfile(path, root.getProject().getCompressor()); + packfile = project.getPackfileManager().openPackfile(path); } else { packfile = null; } - final var project = root.getProject(); final var manager = project.getPackfileManager(); - final var changes = manager.getPackfiles().stream() + final var changes = manager.getArchives().stream() .filter(Packfile::hasChanges) .flatMap(p -> p.getChanges().entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -361,7 +360,7 @@ private void updateExistingPackfiles( ) throws IOException { final var project = root.getProject(); final var manager = project.getPackfileManager(); - final var changes = manager.getPackfiles().stream() + final var changes = manager.getArchives().stream() .filter(Packfile::hasChanges) .collect(Collectors.toMap( Function.identity(), @@ -387,7 +386,7 @@ private void write(@NotNull ProgressMonitor monitor, @NotNull Path path, @Nullab .map(FilePath::hash) .collect(Collectors.toSet()); - for (PackfileBase.FileEntry file : target.getFileEntries()) { + for (Packfile.FileEntry file : target.getFileEntries()) { if (!hashes.contains(file.hash())) { writer.add(new PackfileResource(target, file)); } @@ -424,7 +423,7 @@ private void write(@NotNull ProgressMonitor monitor, @NotNull Path path, @Nullab private static void refreshPackfiles(@NotNull ProgressMonitor monitor, @NotNull Project project) { try (var task = monitor.begin("Refresh packfiles")) { - for (Packfile packfile : project.getPackfileManager().getPackfiles()) { + for (Packfile packfile : project.getPackfileManager().getArchives()) { if (task.isCanceled()) { return; } diff --git a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/NavigatorView.java b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/NavigatorView.java index d7951a225..18bcd6a4e 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/NavigatorView.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/NavigatorView.java @@ -3,6 +3,10 @@ import com.shade.decima.model.app.ProjectChangeListener; import com.shade.decima.model.app.ProjectContainer; import com.shade.decima.model.app.ProjectManager; +import com.shade.decima.model.packfile.Packfile; +import com.shade.decima.model.packfile.PackfileChangeListener; +import com.shade.decima.model.packfile.edit.Change; +import com.shade.decima.model.util.FilePath; import com.shade.decima.ui.menu.MenuConstants; import com.shade.decima.ui.navigator.dnd.NodeTransferHandler; import com.shade.decima.ui.navigator.impl.NavigatorNode; @@ -10,6 +14,7 @@ import com.shade.decima.ui.navigator.impl.NavigatorProjectsNode; import com.shade.decima.ui.views.BaseView; import com.shade.platform.model.messages.MessageBus; +import com.shade.platform.model.messages.MessageBusConnection; import com.shade.platform.model.runtime.VoidProgressMonitor; import com.shade.platform.model.util.IOUtils; import com.shade.platform.ui.editors.EditorManager; @@ -71,7 +76,8 @@ protected NavigatorTree createComponentImpl() { default -> null; }); - MessageBus.getInstance().connect().subscribe(ProjectManager.PROJECTS, new ProjectChangeListener() { + final MessageBusConnection bus = MessageBus.getInstance().connect(); + bus.subscribe(ProjectManager.PROJECTS, new ProjectChangeListener() { @Override public void projectAdded(@NotNull ProjectContainer container) { final var model = tree.getModel(); @@ -110,6 +116,20 @@ public void projectClosed(@NotNull ProjectContainer container) { model.unloadNode(projectNode); } }); + bus.subscribe(Packfile.CHANGES, new PackfileChangeListener() { + @Override + public void fileChanged(@NotNull Packfile packfile, @NotNull FilePath path, @NotNull Change change) { + ProjectManager.getInstance().getOpenProjects().stream() + .filter(p -> p.getPackfileManager() == packfile.getManager()) + .findFirst() + .ifPresent(project -> { + final NavigatorTreeModel model = tree.getModel(); + model + .findFileNode(new VoidProgressMonitor(), NavigatorPath.of(project.getContainer(), packfile, path)) + .whenComplete((node, exception) -> model.fireNodesChanged(node)); + }); + } + }); return tree; } diff --git a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfileNode.java b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfileNode.java index 8d2ef3b84..f99323b8a 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfileNode.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfileNode.java @@ -2,7 +2,6 @@ import com.shade.decima.model.app.Project; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileBase; import com.shade.decima.model.util.FilePath; import com.shade.decima.ui.navigator.NavigatorPath; import com.shade.platform.model.runtime.ProgressMonitor; @@ -34,7 +33,7 @@ protected NavigatorNode[] loadChildren(@NotNull ProgressMonitor monitor) throws try (Stream allFiles = project.listAllFiles()) { allFiles.forEach(path -> { - final long hash = PackfileBase.getPathHash(path); + final long hash = Packfile.getPathHash(path); if (packfile.contains(hash)) { files.add(new FilePath(path.split("/"), hash)); containing.add(hash); @@ -42,7 +41,7 @@ protected NavigatorNode[] loadChildren(@NotNull ProgressMonitor monitor) throws }); } - for (PackfileBase.FileEntry entry : packfile.getFileEntries()) { + for (Packfile.FileEntry entry : packfile.getFileEntries()) { if (!containing.contains(entry.hash())) { files.add(new FilePath(new String[]{"", Long.toHexString(entry.hash())}, entry.hash())); } diff --git a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfilesNode.java b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfilesNode.java index 7307d62d2..04a663d30 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfilesNode.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorPackfilesNode.java @@ -51,7 +51,7 @@ public boolean contains(@NotNull NavigatorPath path) { } @NotNull - public Packfile[] getPackfiles() { + public Packfile[] getArchives() { return packfiles; } } diff --git a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorProjectNode.java b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorProjectNode.java index eafa73ba7..800c6be0b 100644 --- a/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorProjectNode.java +++ b/modules/decima-ui/src/main/java/com/shade/decima/ui/navigator/impl/NavigatorProjectNode.java @@ -4,22 +4,15 @@ import com.shade.decima.model.app.ProjectContainer; import com.shade.decima.model.app.ProjectManager; import com.shade.decima.model.packfile.Packfile; -import com.shade.decima.model.packfile.PackfileChangeListener; -import com.shade.decima.model.packfile.PackfileManager; -import com.shade.decima.ui.Application; import com.shade.decima.ui.navigator.NavigatorPath; import com.shade.decima.ui.navigator.NavigatorSettings; -import com.shade.decima.ui.navigator.NavigatorTreeModel; import com.shade.platform.model.runtime.ProgressMonitor; -import com.shade.platform.model.runtime.VoidProgressMonitor; import com.shade.util.NotNull; import com.shade.util.Nullable; import javax.swing.*; import java.io.IOException; import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; public class NavigatorProjectNode extends NavigatorNode { private final ProjectContainer container; @@ -71,30 +64,13 @@ public String getDescription() { protected NavigatorNode[] loadChildren(@NotNull ProgressMonitor monitor) throws IOException { open(); - final PackfileManager manager = project.getPackfileManager(); - - final PackfileChangeListener listener = (packfile, path, change) -> { - final NavigatorTreeModel model = Application.getNavigator().getModel(); - - model - .findFileNode(new VoidProgressMonitor(), NavigatorPath.of(container, packfile, path)) - .whenComplete((node, exception) -> model.fireNodesChanged(node)); - }; - - final Stream stream = manager.getPackfiles().stream() - .filter(packfile -> !packfile.isEmpty()) - .peek(packfile -> packfile.addChangeListener(listener)); - if (getPackfileView() == NavigatorSettings.PackfileView.GROUPED) { - final Map> groups = stream.collect( - Collectors.groupingBy( - Packfile::getName, - LinkedHashMap::new, - Collectors.toList() - )); + final Map> groups = new LinkedHashMap<>(); + for (Packfile packfile : project.getPackfileManager().getArchives()) { + groups.computeIfAbsent(packfile.getName(), k -> new ArrayList<>()).add(packfile); + } final List children = new ArrayList<>(); - for (Map.Entry> entry : groups.entrySet()) { final String name = entry.getKey(); final List packfiles = entry.getValue(); @@ -110,7 +86,7 @@ protected NavigatorNode[] loadChildren(@NotNull ProgressMonitor monitor) throws return children.toArray(NavigatorNode[]::new); } else { - return stream + return project.getPackfileManager().getArchives().stream() .map(packfile -> new NavigatorPackfileNode(this, packfile)) .toArray(NavigatorNode[]::new); } diff --git a/modules/decima-ui/src/test/java/com/shade/decima/model/packfile/PackfileWriterTest.java b/modules/decima-ui/src/test/java/com/shade/decima/model/packfile/PackfileWriterTest.java index 14f9be1d6..a5208b8cf 100644 --- a/modules/decima-ui/src/test/java/com/shade/decima/model/packfile/PackfileWriterTest.java +++ b/modules/decima-ui/src/test/java/com/shade/decima/model/packfile/PackfileWriterTest.java @@ -62,7 +62,9 @@ public void writePackfileTest(int length) throws IOException { } } - try (Packfile packfile = new Packfile(file, oodle)) { + final var manager = new PackfileManager(oodle); + + try (Packfile packfile = manager.openPackfile(file)) { Assertions.assertEquals(FILES_COUNT, packfile.getFileEntries().size()); for (int i = 0; i < FILES_COUNT; i++) {