001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * https://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.codec.digest; 019 020import java.io.ByteArrayOutputStream; 021import java.io.IOException; 022import java.io.InputStream; 023import java.nio.charset.StandardCharsets; 024import java.nio.file.DirectoryStream; 025import java.nio.file.Files; 026import java.nio.file.Path; 027import java.security.MessageDigest; 028import java.util.HashMap; 029import java.util.Map; 030import java.util.Objects; 031import java.util.Set; 032import java.util.TreeSet; 033import java.util.function.Supplier; 034 035/** 036 * Computes <a href="https://git-scm.com/">Git</a> object identifiers and their generalizations described by the 037 * <a href="https://www.swhid.org/swhid-specification/">SWHID specification</a>. 038 * 039 * <p> 040 * When the hash algorithm is SHA-1, the identifiers produced by this class are identical to those used by Git. 041 * Other hash algorithms produce generalized identifiers as described by the SWHID specification. 042 * </p> 043 * 044 * <p> 045 * Git and SWHID treat file names and symbolic link targets as opaque byte sequences with no defined encoding. The 046 * identifiers produced here coincide with Git's or SWHID's own identifiers only if the original names and targets were 047 * UTF-8 encoded. 048 * </p> 049 * 050 * <p> 051 * This class is immutable and thread-safe. However, the {@link MessageDigest} instances passed to it generally won't be. 052 * </p> 053 * 054 * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a> 055 * @see <a href="https://www.swhid.org/swhid-specification/">SWHID Specification</a> 056 * @since 1.22.0 057 */ 058public class GitIdentifiers { 059 060 /** 061 * Represents a single entry in a Git tree object. 062 * 063 * <p> 064 * A Git tree object encodes a directory snapshot. Each entry holds: 065 * </p> 066 * <ul> 067 * <li>a {@link FileMode} that determines the Unix file mode (e.g. {@code 100644} for a regular file),</li> 068 * <li>the entry name (file or directory name, without a path separator),</li> 069 * <li>the raw object id of the referenced blob or sub-tree.</li> 070 * </ul> 071 * 072 * <p> 073 * Entries are ordered by {@link #compareTo} using Git's tree-sort rule: names are compared as unsigned UTF-8 bytes, and directory names are compared as if 074 * they ended with {@code '/'}, so that {@code foo/} sorts after {@code foobar}. Comparing the UTF-8 bytes rather than the Java {@link String} matches Git's 075 * order for names outside the Basic Multilingual Plane, whose UTF-16 code units do not sort in code point order. 076 * </p> 077 * 078 * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a> 079 * @see <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID Directory Identifier</a> 080 */ 081 static class DirectoryEntry implements Comparable<DirectoryEntry> { 082 083 private static String requireValidName(final String name) { 084 Objects.requireNonNull(name, "name"); 085 if (name.isEmpty() || ".".equals(name) || "..".equals(name)) { 086 throw new IllegalArgumentException("Entry name must not be empty, '.' or '..'"); 087 } 088 if (name.indexOf('/') >= 0 || name.indexOf('\0') >= 0) { 089 throw new IllegalArgumentException("Entry name must not contain '/' or NUL"); 090 } 091 if (!StandardCharsets.UTF_8.newEncoder().canEncode(name)) { 092 throw new IllegalArgumentException("Entry name must not contain unpaired surrogates"); 093 } 094 return name; 095 } 096 097 /** 098 * The entry name (file or directory name, no path separator). 099 */ 100 private final String name; 101 102 /** 103 * The raw object id of the referenced blob or sub-tree. 104 */ 105 private final byte[] rawObjectId; 106 107 /** 108 * The key used for ordering entries within a tree object, as the UTF-8 bytes Git itself compares. 109 * 110 * <p> 111 * Git appends {@code '/'} to directory names before comparing. 112 * </p> 113 */ 114 private final byte[] sortKey; 115 116 /** 117 * The Git object type, which determines the Unix file-mode prefix. 118 */ 119 private final FileMode type; 120 121 /** 122 * Constructs a new entry. 123 * 124 * @param name The nonempty entry name, not {@code .} or {@code ..}, without {@code '/'}, NUL or unpaired surrogates. 125 * @param type The type of the entry, not null. 126 * @param rawObjectId The id of the entry, not null. 127 */ 128 DirectoryEntry(final String name, final FileMode type, final byte[] rawObjectId) { 129 this.name = requireValidName(name); 130 this.type = Objects.requireNonNull(type, "type"); 131 this.sortKey = (type == FileMode.DIRECTORY ? name + "/" : name).getBytes(StandardCharsets.UTF_8); 132 this.rawObjectId = Objects.requireNonNull(rawObjectId, "rawObjectId"); 133 } 134 135 @Override 136 public int compareTo(final DirectoryEntry o) { 137 final byte[] a = sortKey; 138 final byte[] b = o.sortKey; 139 final int shared = Math.min(a.length, b.length); 140 for (int i = 0; i < shared; i++) { 141 final int diff = (a[i] & 0xff) - (b[i] & 0xff); 142 if (diff != 0) { 143 return diff; 144 } 145 } 146 return a.length != b.length ? a.length - b.length : name.compareTo(o.name); 147 } 148 149 @Override 150 public boolean equals(final Object obj) { 151 if (obj == this) { 152 return true; 153 } 154 if (!(obj instanceof DirectoryEntry)) { 155 return false; 156 } 157 final DirectoryEntry other = (DirectoryEntry) obj; 158 return name.equals(other.name); 159 } 160 161 @Override 162 public int hashCode() { 163 return name.hashCode(); 164 } 165 166 } 167 168 /** 169 * The type of a Git tree entry, which maps to a Unix file-mode string. 170 * 171 * <p> 172 * Git encodes the file type and permission bits as an ASCII octal string that precedes the entry name in the binary tree format. The values defined here 173 * cover the four entry types that Git itself produces. 174 * </p> 175 * 176 * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a> 177 */ 178 public enum FileMode { 179 180 /** 181 * A subdirectory. Subdirectories can only be specified by SHA or through a tree mark set with {@code --import-marks}. 182 * 183 * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a> 184 */ 185 DIRECTORY(new byte[] { '4', '0', '0', '0', '0' }), 186 187 /** 188 * A regular, but executable, file. 189 */ 190 EXECUTABLE(new byte[] { '1', '0', '0', '7', '5', '5' }), 191 192 /** 193 * A gitlink, SHA-1 of the object refers to a commit in another repository. Git links can only be specified either by SHA or through a commit mark. They 194 * are used to implement submodules. 195 * 196 * @see <a href="https://git-scm.com/docs/gitdatamodel">gitdatamodel - Git's core data model</a> 197 * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a> 198 */ 199 GIT_LINK(new byte[] { '1', '6', '0', '0', '0', '0' }), 200 201 /** 202 * A regular (non-executable) file. 203 * <p> 204 * The majority of files in most projects use this mode. If in doubt, this is what you want. 205 * </p> 206 */ 207 REGULAR(new byte[] { '1', '0', '0', '6', '4', '4' }), 208 209 /** 210 * A symbolic link. The content of the file will be the link target. 211 */ 212 SYMBOLIC_LINK(new byte[] { '1', '2', '0', '0', '0', '0' }); 213 214 private static FileMode get(final Path path) { 215 // Symbolic links first 216 if (Files.isSymbolicLink(path)) { 217 return SYMBOLIC_LINK; 218 } 219 if (Files.isDirectory(path)) { 220 return DIRECTORY; 221 } 222 if (Files.isExecutable(path)) { 223 return EXECUTABLE; 224 } 225 return REGULAR; 226 } 227 228 /** 229 * Serialized {@code mode}: since this is mutable, it must remain private. 230 */ 231 private final byte[] modeBytes; 232 233 FileMode(final byte[] modeBytes) { 234 // No need for a defensive copy since the array is private and never exposed, 235 this.modeBytes = modeBytes; 236 } 237 } 238 239 /** 240 * Builds a Git tree identifier for a virtual directory structure, such as the contents of 241 * an archive. 242 */ 243 public static final class TreeIdBuilder implements Supplier<byte[]> { 244 245 /** 246 * Supplies a blob identifier that may throw {@link IOException}. 247 */ 248 @FunctionalInterface 249 private interface BlobIdSupplier { 250 byte[] get() throws IOException; 251 } 252 253 private final Map<String, TreeIdBuilder> dirEntries = new HashMap<>(); 254 private final Map<String, DirectoryEntry> fileEntries = new HashMap<>(); 255 private final MessageDigest messageDigest; 256 257 private TreeIdBuilder(final MessageDigest messageDigest) { 258 this.messageDigest = Objects.requireNonNull(messageDigest, "messageDigest"); 259 } 260 261 /** 262 * Adds and returns the {@link TreeIdBuilder} for the named subdirectory, creating it if absent. 263 * 264 * @param name The relative path of the subdirectory in normalized form (may contain {@code '/'}). 265 * @return The {@link TreeIdBuilder} for the subdirectory. 266 * @throws IllegalArgumentException Thrown if any path component is {@code ".."}, contains NUL or contains unpaired surrogates. 267 */ 268 public TreeIdBuilder addDirectory(final String name) { 269 TreeIdBuilder current = this; 270 for (final String component : name.split("/", -1)) { 271 // Noop segments 272 if (component.isEmpty() || ".".equals(component)) { 273 continue; 274 } 275 current = current.dirEntries.computeIfAbsent(DirectoryEntry.requireValidName(component), k -> new TreeIdBuilder(messageDigest)); 276 } 277 return current; 278 } 279 280 private void addFile(final FileMode mode, final String name, final BlobIdSupplier blobId) throws IOException { 281 final int slash = name.lastIndexOf('/'); 282 if (slash < 0) { 283 DirectoryEntry.requireValidName(name); 284 fileEntries.put(name, new DirectoryEntry(name, mode, blobId.get())); 285 } else { 286 addDirectory(name.substring(0, slash)).addFile(mode, name.substring(slash + 1), blobId); 287 } 288 } 289 290 /** 291 * Adds a file entry at the given path within this tree. 292 * 293 * <p> 294 * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically. 295 * </p> 296 * 297 * @param mode The file mode (e.g. {@link FileMode#REGULAR}). 298 * @param name The relative path of the entry in normalized form(may contain {@code '/'}). 299 * @param data The file content. 300 * @throws IOException Thrown if an I/O error occurs. 301 * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired 302 * surrogates. 303 */ 304 public void addFile(final FileMode mode, final String name, final byte[] data) throws IOException { 305 addFile(mode, name, () -> blobId(messageDigest, data)); 306 } 307 308 /** 309 * Adds a file entry at the given path within this tree, streaming content without buffering. 310 * 311 * <p> 312 * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically. 313 * </p> 314 * 315 * <p> 316 * The stream is eagerly drained. 317 * </p> 318 * 319 * @param mode The file mode (e.g. {@link FileMode#REGULAR}). 320 * @param name The relative path of the entry in normalized form(may contain {@code '/'}). 321 * @param dataSize The exact number of bytes in {@code data}. 322 * @param data The file content. 323 * @throws IOException Thrown if the stream cannot be read, or does not contain exactly {@code dataSize} bytes. 324 * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired 325 * surrogates. 326 */ 327 public void addFile(final FileMode mode, final String name, final long dataSize, final InputStream data) throws IOException { 328 addFile(mode, name, () -> blobId(messageDigest, dataSize, data)); 329 } 330 331 /** 332 * Adds a symbolic link entry at the given path within this tree. 333 * 334 * <p> 335 * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically. 336 * </p> 337 * 338 * @param name The relative path of the entry in normalized form(may contain {@code '/'}). 339 * @param target The target of the symbolic link. 340 * @throws IOException Thrown if an I/O error occurs. 341 * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired 342 * surrogates. 343 */ 344 public void addSymbolicLink(final String name, final String target) throws IOException { 345 addFile(FileMode.SYMBOLIC_LINK, name, target.getBytes(StandardCharsets.UTF_8)); 346 } 347 348 /** 349 * Gets the Git tree identifier for this directory and all its descendants. 350 * 351 * @return The raw tree identifier bytes. 352 * @throws IllegalStateException Thrown if a file and a directory have the same name in this directory or any descendant. 353 */ 354 @Override 355 public byte[] get() { 356 for (final String name : dirEntries.keySet()) { 357 if (fileEntries.containsKey(name)) { 358 throw new IllegalStateException("File and directory have the same name: " + name); 359 } 360 } 361 final Set<DirectoryEntry> entries = new TreeSet<>(fileEntries.values()); 362 dirEntries.forEach((k, v) -> entries.add(new DirectoryEntry(k, FileMode.DIRECTORY, v.get()))); 363 final ByteArrayOutputStream baos = new ByteArrayOutputStream(); 364 for (final DirectoryEntry entry : entries) { 365 baos.write(entry.type.modeBytes, 0, entry.type.modeBytes.length); 366 baos.write(' '); 367 final byte[] bytes = entry.name.getBytes(StandardCharsets.UTF_8); 368 baos.write(bytes, 0, bytes.length); 369 baos.write('\0'); 370 baos.write(entry.rawObjectId, 0, entry.rawObjectId.length); 371 } 372 messageDigest.reset(); 373 DigestUtils.updateDigest(messageDigest, getGitTreePrefix(baos.size())); 374 return DigestUtils.updateDigest(messageDigest, baos.toByteArray()).digest(); 375 } 376 377 private TreeIdBuilder populate(final Path directory) throws IOException { 378 try (DirectoryStream<Path> files = Files.newDirectoryStream(directory)) { 379 for (final Path path : files) { 380 final String name = Objects.toString(path.getFileName()); 381 final FileMode mode = FileMode.get(path); 382 if (mode == FileMode.DIRECTORY) { 383 addDirectory(name).populate(path); 384 } else { 385 addFile(mode, name, () -> blobId(messageDigest, path)); 386 } 387 } 388 } 389 return this; 390 } 391 } 392 393 /** 394 * Reads through a byte array and returns a generalized Git blob identifier. 395 * 396 * <p> 397 * The identifier is computed in the way described by the 398 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash 399 * algorithm. 400 * </p> 401 * 402 * <p> 403 * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier. 404 * </p> 405 * 406 * @param messageDigest The MessageDigest to use (for example SHA-1). 407 * @param data Data to digest. 408 * @return A generalized Git blob identifier. 409 */ 410 public static byte[] blobId(final MessageDigest messageDigest, final byte[] data) { 411 messageDigest.reset(); 412 DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(data.length)); 413 return DigestUtils.digest(messageDigest, data); 414 } 415 416 /** 417 * Reads through a stream of known size and returns a generalized Git blob identifier, without buffering. 418 * 419 * <p> 420 * When the size of the content is known in advance, this overload streams {@code data} directly through 421 * the digest without buffering the full content in memory. 422 * </p> 423 * 424 * <p> 425 * The stream is drained to its end. If the number of bytes read differs from {@code dataSize}, an {@link IOException} is thrown. 426 * </p> 427 * 428 * <p> 429 * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier. 430 * </p> 431 * 432 * @param messageDigest The MessageDigest to use (for example SHA-1). 433 * @param dataSize The exact number of bytes in {@code data}. 434 * @param data Stream to digest. 435 * @return A generalized Git blob identifier. 436 * @throws IOException Thrown on error reading the stream, or if the stream does not contain exactly {@code dataSize} bytes. 437 */ 438 public static byte[] blobId(final MessageDigest messageDigest, final long dataSize, final InputStream data) throws IOException { 439 messageDigest.reset(); 440 DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(dataSize)); 441 final byte[] buffer = new byte[8192]; 442 long actualSize = 0; 443 int read; 444 while ((read = data.read(buffer)) != -1) { 445 messageDigest.update(buffer, 0, read); 446 actualSize += read; 447 } 448 if (actualSize != dataSize) { 449 throw new IOException("Stream contained " + actualSize + " bytes, but dataSize declared " + dataSize + " bytes"); 450 } 451 return messageDigest.digest(); 452 } 453 454 /** 455 * Reads through a file and returns a generalized Git blob identifier. 456 * 457 * <p> 458 * The identifier is computed in the way described by the 459 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash 460 * algorithm. 461 * </p> 462 * 463 * <p> 464 * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier. 465 * </p> 466 * 467 * @param messageDigest The MessageDigest to use (for example SHA-1). 468 * @param data Path to the file to digest. 469 * @return A generalized Git blob identifier. 470 * @throws IOException Thrown on error accessing the file, or if the number of bytes read differs from its measured size. 471 */ 472 public static byte[] blobId(final MessageDigest messageDigest, final Path data) throws IOException { 473 if (Files.isSymbolicLink(data)) { 474 final byte[] linkTarget = Files.readSymbolicLink(data).toString().getBytes(StandardCharsets.UTF_8); 475 return blobId(messageDigest, linkTarget); 476 } 477 final long dataSize = Files.size(data); 478 try (InputStream input = Files.newInputStream(data)) { 479 return blobId(messageDigest, dataSize, input); 480 } 481 } 482 483 private static byte[] getGitBlobPrefix(final long dataSize) { 484 return getGitPrefix("blob", dataSize); 485 } 486 487 private static byte[] getGitPrefix(final String type, final long dataSize) { 488 return (type + " " + dataSize + "\0").getBytes(StandardCharsets.UTF_8); 489 } 490 491 private static byte[] getGitTreePrefix(final long dataSize) { 492 return getGitPrefix("tree", dataSize); 493 } 494 495 /** 496 * Reads through a directory and returns a generalized Git tree identifier. 497 * 498 * <p> 499 * The identifier is computed in the way described by the 500 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash 501 * algorithm. 502 * </p> 503 * 504 * <p> 505 * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier. 506 * </p> 507 * 508 * @param messageDigest The MessageDigest to use (for example SHA-1). 509 * @param data Path to the directory to digest. 510 * @return A generalized Git tree identifier. 511 * @throws IOException Thrown on error accessing the directory or its contents. 512 */ 513 public static byte[] treeId(final MessageDigest messageDigest, final Path data) throws IOException { 514 return treeIdBuilder(messageDigest).populate(data).get(); 515 } 516 517 /** 518 * Returns a new {@link TreeIdBuilder} for constructing a generalized Git tree identifier from a virtual directory 519 * structure, such as the contents of an archive. 520 * 521 * <p> 522 * The identifier is computed in the way described by the 523 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash 524 * algorithm. 525 * </p> 526 * 527 * <p> 528 * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier. 529 * </p> 530 * 531 * @param messageDigest The MessageDigest to use (for example SHA-1). 532 * @return A new {@link TreeIdBuilder}. 533 */ 534 public static TreeIdBuilder treeIdBuilder(final MessageDigest messageDigest) { 535 return new TreeIdBuilder(messageDigest); 536 } 537 538 private GitIdentifiers() { 539 // utility class 540 } 541}