001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      https://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017
018package org.apache.commons.codec.digest;
019
020import java.io.ByteArrayOutputStream;
021import java.io.IOException;
022import java.io.InputStream;
023import java.nio.charset.StandardCharsets;
024import java.nio.file.DirectoryStream;
025import java.nio.file.Files;
026import java.nio.file.Path;
027import java.security.MessageDigest;
028import java.util.HashMap;
029import java.util.Map;
030import java.util.Objects;
031import java.util.Set;
032import java.util.TreeSet;
033import java.util.function.Supplier;
034
035/**
036 * Computes <a href="https://git-scm.com/">Git</a> object identifiers and their generalizations described by the
037 * <a href="https://www.swhid.org/swhid-specification/">SWHID specification</a>.
038 *
039 * <p>
040 * When the hash algorithm is SHA-1, the identifiers produced by this class are identical to those used by Git.
041 * Other hash algorithms produce generalized identifiers as described by the SWHID specification.
042 * </p>
043 *
044 * <p>
045 * Git and SWHID treat file names and symbolic link targets as opaque byte sequences with no defined encoding. The
046 * identifiers produced here coincide with Git's or SWHID's own identifiers only if the original names and targets were
047 * UTF-8 encoded.
048 * </p>
049 *
050 * <p>
051 * This class is immutable and thread-safe. However, the {@link MessageDigest} instances passed to it generally won't be.
052 * </p>
053 *
054 * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
055 * @see <a href="https://www.swhid.org/swhid-specification/">SWHID Specification</a>
056 * @since 1.22.0
057 */
058public class GitIdentifiers {
059
060    /**
061     * Represents a single entry in a Git tree object.
062     *
063     * <p>
064     * A Git tree object encodes a directory snapshot. Each entry holds:
065     * </p>
066     * <ul>
067     *   <li>a {@link FileMode} that determines the Unix file mode (e.g. {@code 100644} for a regular file),</li>
068     *   <li>the entry name (file or directory name, without a path separator),</li>
069     *   <li>the raw object id of the referenced blob or sub-tree.</li>
070     * </ul>
071     *
072     * <p>
073     * Entries are ordered by {@link #compareTo} using Git's tree-sort rule: names are compared as unsigned UTF-8 bytes, and directory names are compared as if
074     * they ended with {@code '/'}, so that {@code foo/} sorts after {@code foobar}. Comparing the UTF-8 bytes rather than the Java {@link String} matches Git's
075     * order for names outside the Basic Multilingual Plane, whose UTF-16 code units do not sort in code point order.
076     * </p>
077     *
078     * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
079     * @see <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID Directory Identifier</a>
080     */
081    static class DirectoryEntry implements Comparable<DirectoryEntry> {
082
083        private static String requireValidName(final String name) {
084            Objects.requireNonNull(name, "name");
085            if (name.isEmpty() || ".".equals(name) || "..".equals(name)) {
086                throw new IllegalArgumentException("Entry name must not be empty, '.' or '..'");
087            }
088            if (name.indexOf('/') >= 0 || name.indexOf('\0') >= 0) {
089                throw new IllegalArgumentException("Entry name must not contain '/' or NUL");
090            }
091            if (!StandardCharsets.UTF_8.newEncoder().canEncode(name)) {
092                throw new IllegalArgumentException("Entry name must not contain unpaired surrogates");
093            }
094            return name;
095        }
096
097        /**
098         * The entry name (file or directory name, no path separator).
099         */
100        private final String name;
101
102        /**
103         * The raw object id of the referenced blob or sub-tree.
104         */
105        private final byte[] rawObjectId;
106
107        /**
108         * The key used for ordering entries within a tree object, as the UTF-8 bytes Git itself compares.
109         *
110         * <p>
111         * Git appends {@code '/'} to directory names before comparing.
112         * </p>
113         */
114        private final byte[] sortKey;
115
116        /**
117         * The Git object type, which determines the Unix file-mode prefix.
118         */
119        private final FileMode type;
120
121        /**
122         * Constructs a new entry.
123         *
124         * @param name The nonempty entry name, not {@code .} or {@code ..}, without {@code '/'}, NUL or unpaired surrogates.
125         * @param type The type of the entry, not null.
126         * @param rawObjectId The id of the entry, not null.
127         */
128        DirectoryEntry(final String name, final FileMode type, final byte[] rawObjectId) {
129            this.name = requireValidName(name);
130            this.type = Objects.requireNonNull(type, "type");
131            this.sortKey = (type == FileMode.DIRECTORY ? name + "/" : name).getBytes(StandardCharsets.UTF_8);
132            this.rawObjectId = Objects.requireNonNull(rawObjectId, "rawObjectId");
133        }
134
135        @Override
136        public int compareTo(final DirectoryEntry o) {
137            final byte[] a = sortKey;
138            final byte[] b = o.sortKey;
139            final int shared = Math.min(a.length, b.length);
140            for (int i = 0; i < shared; i++) {
141                final int diff = (a[i] & 0xff) - (b[i] & 0xff);
142                if (diff != 0) {
143                    return diff;
144                }
145            }
146            return a.length != b.length ? a.length - b.length : name.compareTo(o.name);
147        }
148
149        @Override
150        public boolean equals(final Object obj) {
151            if (obj == this) {
152                return true;
153            }
154            if (!(obj instanceof DirectoryEntry)) {
155                return false;
156            }
157            final DirectoryEntry other = (DirectoryEntry) obj;
158            return name.equals(other.name);
159        }
160
161        @Override
162        public int hashCode() {
163            return name.hashCode();
164        }
165
166    }
167
168    /**
169     * The type of a Git tree entry, which maps to a Unix file-mode string.
170     *
171     * <p>
172     * Git encodes the file type and permission bits as an ASCII octal string that precedes the entry name in the binary tree format. The values defined here
173     * cover the four entry types that Git itself produces.
174     * </p>
175     *
176     * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
177     */
178    public enum FileMode {
179
180        /**
181         * A subdirectory. Subdirectories can only be specified by SHA or through a tree mark set with {@code --import-marks}.
182         *
183         * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a>
184         */
185        DIRECTORY(new byte[] { '4', '0', '0', '0', '0' }),
186
187        /**
188         * A regular, but executable, file.
189         */
190        EXECUTABLE(new byte[] { '1', '0', '0', '7', '5', '5' }),
191
192        /**
193         * A gitlink, SHA-1 of the object refers to a commit in another repository. Git links can only be specified either by SHA or through a commit mark. They
194         * are used to implement submodules.
195         *
196         * @see <a href="https://git-scm.com/docs/gitdatamodel">gitdatamodel - Git&#39;s core data model</a>
197         * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a>
198         */
199        GIT_LINK(new byte[] { '1', '6', '0', '0', '0', '0' }),
200
201        /**
202         * A regular (non-executable) file.
203         * <p>
204         * The majority of files in most projects use this mode. If in doubt, this is what you want.
205         * </p>
206         */
207        REGULAR(new byte[] { '1', '0', '0', '6', '4', '4' }),
208
209        /**
210         * A symbolic link. The content of the file will be the link target.
211         */
212        SYMBOLIC_LINK(new byte[] { '1', '2', '0', '0', '0', '0' });
213
214        private static FileMode get(final Path path) {
215            // Symbolic links first
216            if (Files.isSymbolicLink(path)) {
217                return SYMBOLIC_LINK;
218            }
219            if (Files.isDirectory(path)) {
220                return DIRECTORY;
221            }
222            if (Files.isExecutable(path)) {
223                return EXECUTABLE;
224            }
225            return REGULAR;
226        }
227
228        /**
229         * Serialized {@code mode}: since this is mutable, it must remain private.
230         */
231        private final byte[] modeBytes;
232
233        FileMode(final byte[] modeBytes) {
234            // No need for a defensive copy since the array is private and never exposed,
235            this.modeBytes = modeBytes;
236        }
237    }
238
239    /**
240     * Builds a Git tree identifier for a virtual directory structure, such as the contents of
241     * an archive.
242     */
243    public static final class TreeIdBuilder implements Supplier<byte[]> {
244
245        /**
246         * Supplies a blob identifier that may throw {@link IOException}.
247         */
248        @FunctionalInterface
249        private interface BlobIdSupplier {
250            byte[] get() throws IOException;
251        }
252
253        private final Map<String, TreeIdBuilder> dirEntries = new HashMap<>();
254        private final Map<String, DirectoryEntry> fileEntries = new HashMap<>();
255        private final MessageDigest messageDigest;
256
257        private TreeIdBuilder(final MessageDigest messageDigest) {
258            this.messageDigest = Objects.requireNonNull(messageDigest, "messageDigest");
259        }
260
261        /**
262         * Adds and returns the {@link TreeIdBuilder} for the named subdirectory, creating it if absent.
263         *
264         * @param name The relative path of the subdirectory in normalized form (may contain {@code '/'}).
265         * @return The {@link TreeIdBuilder} for the subdirectory.
266         * @throws IllegalArgumentException Thrown if any path component is {@code ".."}, contains NUL or contains unpaired surrogates.
267         */
268        public TreeIdBuilder addDirectory(final String name) {
269            TreeIdBuilder current = this;
270            for (final String component : name.split("/", -1)) {
271                // Noop segments
272                if (component.isEmpty() || ".".equals(component)) {
273                    continue;
274                }
275                current = current.dirEntries.computeIfAbsent(DirectoryEntry.requireValidName(component), k -> new TreeIdBuilder(messageDigest));
276            }
277            return current;
278        }
279
280        private void addFile(final FileMode mode, final String name, final BlobIdSupplier blobId) throws IOException {
281            final int slash = name.lastIndexOf('/');
282            if (slash < 0) {
283                DirectoryEntry.requireValidName(name);
284                fileEntries.put(name, new DirectoryEntry(name, mode, blobId.get()));
285            } else {
286                addDirectory(name.substring(0, slash)).addFile(mode, name.substring(slash + 1), blobId);
287            }
288        }
289
290        /**
291         * Adds a file entry at the given path within this tree.
292         *
293         * <p>
294         * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
295         * </p>
296         *
297         * @param mode The file mode (e.g. {@link FileMode#REGULAR}).
298         * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
299         * @param data The file content.
300         * @throws IOException Thrown if an I/O error occurs.
301         * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
302         *                                  surrogates.
303         */
304        public void addFile(final FileMode mode, final String name, final byte[] data) throws IOException {
305            addFile(mode, name, () -> blobId(messageDigest, data));
306        }
307
308        /**
309         * Adds a file entry at the given path within this tree, streaming content without buffering.
310         *
311         * <p>
312         * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
313         * </p>
314         *
315         * <p>
316         * The stream is eagerly drained.
317         * </p>
318         *
319         * @param mode     The file mode (e.g. {@link FileMode#REGULAR}).
320         * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
321         * @param dataSize The exact number of bytes in {@code data}.
322         * @param data     The file content.
323         * @throws IOException Thrown if the stream cannot be read, or does not contain exactly {@code dataSize} bytes.
324         * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
325         *                                  surrogates.
326         */
327        public void addFile(final FileMode mode, final String name, final long dataSize, final InputStream data) throws IOException {
328            addFile(mode, name, () -> blobId(messageDigest, dataSize, data));
329        }
330
331        /**
332         * Adds a symbolic link entry at the given path within this tree.
333         *
334         * <p>
335         * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
336         * </p>
337         *
338         * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
339         * @param target The target of the symbolic link.
340         * @throws IOException Thrown if an I/O error occurs.
341         * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
342         *                                  surrogates.
343         */
344        public void addSymbolicLink(final String name, final String target) throws IOException {
345            addFile(FileMode.SYMBOLIC_LINK, name, target.getBytes(StandardCharsets.UTF_8));
346        }
347
348        /**
349         * Gets the Git tree identifier for this directory and all its descendants.
350         *
351         * @return The raw tree identifier bytes.
352         * @throws IllegalStateException Thrown if a file and a directory have the same name in this directory or any descendant.
353         */
354        @Override
355        public byte[] get() {
356            for (final String name : dirEntries.keySet()) {
357                if (fileEntries.containsKey(name)) {
358                    throw new IllegalStateException("File and directory have the same name: " + name);
359                }
360            }
361            final Set<DirectoryEntry> entries = new TreeSet<>(fileEntries.values());
362            dirEntries.forEach((k, v) -> entries.add(new DirectoryEntry(k, FileMode.DIRECTORY, v.get())));
363            final ByteArrayOutputStream baos = new ByteArrayOutputStream();
364            for (final DirectoryEntry entry : entries) {
365                baos.write(entry.type.modeBytes, 0, entry.type.modeBytes.length);
366                baos.write(' ');
367                final byte[] bytes = entry.name.getBytes(StandardCharsets.UTF_8);
368                baos.write(bytes, 0, bytes.length);
369                baos.write('\0');
370                baos.write(entry.rawObjectId, 0, entry.rawObjectId.length);
371            }
372            messageDigest.reset();
373            DigestUtils.updateDigest(messageDigest, getGitTreePrefix(baos.size()));
374            return DigestUtils.updateDigest(messageDigest, baos.toByteArray()).digest();
375        }
376
377        private TreeIdBuilder populate(final Path directory) throws IOException {
378            try (DirectoryStream<Path> files = Files.newDirectoryStream(directory)) {
379                for (final Path path : files) {
380                    final String name = Objects.toString(path.getFileName());
381                    final FileMode mode = FileMode.get(path);
382                    if (mode == FileMode.DIRECTORY) {
383                        addDirectory(name).populate(path);
384                    } else {
385                        addFile(mode, name, () -> blobId(messageDigest, path));
386                    }
387                }
388            }
389            return this;
390        }
391    }
392
393    /**
394     * Reads through a byte array and returns a generalized Git blob identifier.
395     *
396     * <p>
397     * The identifier is computed in the way described by the
398     * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash
399     * algorithm.
400     * </p>
401     *
402     * <p>
403     * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
404     * </p>
405     *
406     * @param messageDigest The MessageDigest to use (for example SHA-1).
407     * @param data          Data to digest.
408     * @return A generalized Git blob identifier.
409     */
410    public static byte[] blobId(final MessageDigest messageDigest, final byte[] data) {
411        messageDigest.reset();
412        DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(data.length));
413        return DigestUtils.digest(messageDigest, data);
414    }
415
416    /**
417     * Reads through a stream of known size and returns a generalized Git blob identifier, without buffering.
418     *
419     * <p>
420     * When the size of the content is known in advance, this overload streams {@code data} directly through
421     * the digest without buffering the full content in memory.
422     * </p>
423     *
424     * <p>
425     * The stream is drained to its end. If the number of bytes read differs from {@code dataSize}, an {@link IOException} is thrown.
426     * </p>
427     *
428     * <p>
429     * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
430     * </p>
431     *
432     * @param messageDigest The MessageDigest to use (for example SHA-1).
433     * @param dataSize      The exact number of bytes in {@code data}.
434     * @param data          Stream to digest.
435     * @return A generalized Git blob identifier.
436     * @throws IOException Thrown on error reading the stream, or if the stream does not contain exactly {@code dataSize} bytes.
437     */
438    public static byte[] blobId(final MessageDigest messageDigest, final long dataSize, final InputStream data) throws IOException {
439        messageDigest.reset();
440        DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(dataSize));
441        final byte[] buffer = new byte[8192];
442        long actualSize = 0;
443        int read;
444        while ((read = data.read(buffer)) != -1) {
445            messageDigest.update(buffer, 0, read);
446            actualSize += read;
447        }
448        if (actualSize != dataSize) {
449            throw new IOException("Stream contained " + actualSize + " bytes, but dataSize declared " + dataSize + " bytes");
450        }
451        return messageDigest.digest();
452    }
453
454    /**
455     * Reads through a file and returns a generalized Git blob identifier.
456     *
457     * <p>
458     * The identifier is computed in the way described by the
459     * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash
460     * algorithm.
461     * </p>
462     *
463     * <p>
464     * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
465     * </p>
466     *
467     * @param messageDigest The MessageDigest to use (for example SHA-1).
468     * @param data          Path to the file to digest.
469     * @return A generalized Git blob identifier.
470     * @throws IOException Thrown on error accessing the file, or if the number of bytes read differs from its measured size.
471     */
472    public static byte[] blobId(final MessageDigest messageDigest, final Path data) throws IOException {
473        if (Files.isSymbolicLink(data)) {
474            final byte[] linkTarget = Files.readSymbolicLink(data).toString().getBytes(StandardCharsets.UTF_8);
475            return blobId(messageDigest, linkTarget);
476        }
477        final long dataSize = Files.size(data);
478        try (InputStream input = Files.newInputStream(data)) {
479            return blobId(messageDigest, dataSize, input);
480        }
481    }
482
483    private static byte[] getGitBlobPrefix(final long dataSize) {
484        return getGitPrefix("blob", dataSize);
485    }
486
487    private static byte[] getGitPrefix(final String type, final long dataSize) {
488        return (type + " " + dataSize + "\0").getBytes(StandardCharsets.UTF_8);
489    }
490
491    private static byte[] getGitTreePrefix(final long dataSize) {
492        return getGitPrefix("tree", dataSize);
493    }
494
495    /**
496     * Reads through a directory and returns a generalized Git tree identifier.
497     *
498     * <p>
499     * The identifier is computed in the way described by the
500     * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash
501     * algorithm.
502     * </p>
503     *
504     * <p>
505     * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
506     * </p>
507     *
508     * @param messageDigest The MessageDigest to use (for example SHA-1).
509     * @param data          Path to the directory to digest.
510     * @return A generalized Git tree identifier.
511     * @throws IOException Thrown on error accessing the directory or its contents.
512     */
513    public static byte[] treeId(final MessageDigest messageDigest, final Path data) throws IOException {
514        return treeIdBuilder(messageDigest).populate(data).get();
515    }
516
517    /**
518     * Returns a new {@link TreeIdBuilder} for constructing a generalized Git tree identifier from a virtual directory
519     * structure, such as the contents of an archive.
520     *
521     * <p>
522     * The identifier is computed in the way described by the
523     * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash
524     * algorithm.
525     * </p>
526     *
527     * <p>
528     * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
529     * </p>
530     *
531     * @param messageDigest The MessageDigest to use (for example SHA-1).
532     * @return A new {@link TreeIdBuilder}.
533     */
534    public static TreeIdBuilder treeIdBuilder(final MessageDigest messageDigest) {
535        return new TreeIdBuilder(messageDigest);
536    }
537
538    private GitIdentifiers() {
539        // utility class
540    }
541}