Subversion Repositories SvarDOS

Rev

Rev 673 | Rev 730 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 673 Rev 719
1 
<?php /*
 1 
<?php /*
2 
 
 2 
 
3 
  SvarDOS repo index builder
 3 
  SvarDOS repo index builder
4 
  Copyright (C) Mateusz Viste 2012-2022
 4 
  Copyright (C) Mateusz Viste 2012-2022
5 
 
 5 
 
6 
  buildidx computes an index tsv file for the SvarDOS repository.
 6 
  buildidx computes an index tsv file for the SvarDOS repository.
7 
  it must be executed pointing to a directory that stores packages (*.svp)
 7 
  it must be executed pointing to a directory that stores packages (*.svp)
8 
  files. buildidx will generate the index file and save it into the package
 8 
  files. buildidx will generate the index file and save it into the package
9 
  repository.
 9 
  repository.
10 
 
 10 
 
11 
  requires php-zip
 11 
  requires php-zip
12 
 
 12 
 
- 
 
 13 
  15 feb 2022: index is generated as json, contains all filenames and alt versions
13 
  14 feb 2022: packages are expected to have the *.svp extension
 14 
  14 feb 2022: packages are expected to have the *.svp extension
14 
  12 feb 2022: skip source packages from being processed (*.src.zip)
 15 
  12 feb 2022: skip source packages from being processed (*.src.zip)
15 
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
 16 
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
16 
  13 feb 2021: 'title' LSM field is no longer looked after
 17 
  13 feb 2021: 'title' LSM field is no longer looked after
17 
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
 18 
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
18 
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
 19 
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
19 
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
 20 
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
20 
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
 21 
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
21 
  27 aug 2016: accepting full paths to repos (starting with /...)
 22 
  27 aug 2016: accepting full paths to repos (starting with /...)
22 
  07 dec 2013: rewritten buildidx in ANSI C89
 23 
  07 dec 2013: rewritten buildidx in ANSI C89
23 
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
 24 
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
24 
  22 jul 2013: creating a listing.txt file with list of packages
 25 
  22 jul 2013: creating a listing.txt file with list of packages
25 
  18 jul 2013: writing the number of packaged into the first line of the lst file
 26 
  18 jul 2013: writing the number of packaged into the first line of the lst file
26 
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
 27 
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
27 
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
 28 
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
28 
  04 feb 2013: added CRC32 support
 29 
  04 feb 2013: added CRC32 support
29 
  22 sep 2012: forked 1st version from FDUPDATE builder
 30 
  22 sep 2012: forked 1st version from FDUPDATE builder
30 
*/
 31 
*/
31 
 
 32 
 
32 
$PVER = "20220214";
 33 
$PVER = "20220215";
33 
 
 34 
 
34 
 
 35 
 
35 
// computes the BSD sum of a file and returns it
 36 
// computes the BSD sum of a file and returns it
36 
function file2bsum($fname) {
 37 
function file2bsum($fname) {
37 
  $result = 0;
 38 
  $result = 0;
38 
 
 39 
 
39 
  $fd = fopen($fname, 'rb');
 40 
  $fd = fopen($fname, 'rb');
40 
  if ($fd === false) return(0);
 41 
  if ($fd === false) return(0);
41 
 
 42 
 
42 
  while (!feof($fd)) {
 43 
  while (!feof($fd)) {
43 
 
 44 
 
44 
    $buff = fread($fd, 1024 * 1024);
 45 
    $buff = fread($fd, 1024 * 1024);
45 
 
 46 
 
46 
    $slen = strlen($buff);
 47 
    $slen = strlen($buff);
47 
    for ($i = 0; $i < $slen; $i++) {
 48 
    for ($i = 0; $i < $slen; $i++) {
48 
      // rotr
 49 
      // rotr
49 
      $result = ($result >> 1) | ($result << 15);
 50 
      $result = ($result >> 1) | ($result << 15);
50 
      // add and truncate to 16 bits
 51 
      // add and truncate to 16 bits
51 
      $result += ord($buff[$i]);
 52 
      $result += ord($buff[$i]);
52 
      $result &= 0xffff;
 53 
      $result &= 0xffff;
53 
    }
 54 
    }
54 
  }
 55 
  }
55 
 
 56 
 
56 
  fclose($fd);
 57 
  fclose($fd);
57 
  return($result);
 58 
  return($result);
58 
}
 59 
}
59 
 
 60 
 
60 
 
 61 
 
61 
// reads file fil from zip archive z and returns its content, or false on error
 62 
// reads file fil from zip archive z and returns its content, or false on error
62 
function read_file_from_zip($z, $fil) {
 63 
function read_file_from_zip($z, $fil) {
63 
  $zip = new ZipArchive;
 64 
  $zip = new ZipArchive;
64 
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
 65 
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
65 
    echo "ERROR: failed to open zip file '{$z}'\n";
 66 
    echo "ERROR: failed to open zip file '{$z}'\n";
66 
    return(false);
 67 
    return(false);
67 
  }
 68 
  }
68 
 
 69 
 
69 
  // load the appinfo/pkgname.lsm file
 70 
  // load the appinfo/pkgname.lsm file
70 
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
 71 
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
71 
 
 72 
 
72 
  $zip->close();
 73 
  $zip->close();
73 
  return($res);
 74 
  return($res);
74 
}
 75 
}
75 
 
 76 
 
76 
 
 77 
 
77 
// reads a LSM string and returns it in the form of an array
 78 
// reads a LSM string and returns it in the form of an array
78 
function parse_lsm($s) {
 79 
function parse_lsm($s) {
79 
  $res = array();
 80 
  $res = array();
80 
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
 81 
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
81 
    // the line is "token: value", let's find the colon
 82 
    // the line is "token: value", let's find the colon
82 
    $colpos = strpos($l, ':');
 83 
    $colpos = strpos($l, ':');
83 
    if (($colpos === false) || ($colpos === 0)) continue;
 84 
    if (($colpos === false) || ($colpos === 0)) continue;
84 
    $tok = strtolower(trim(substr($l, 0, $colpos)));
 85 
    $tok = strtolower(trim(substr($l, 0, $colpos)));
85 
    $val = trim(substr($l, $colpos + 1));
 86 
    $val = trim(substr($l, $colpos + 1));
86 
    $res[$tok] = $val;
 87 
    $res[$tok] = $val;
87 
  }
 88 
  }
88 
  return($res);
 89 
  return($res);
89 
}
 90 
}
90 
 
 91 
 
91 
 
 92 
 
92 
// ***************** MAIN ROUTINE *********************************************
 93 
// ***************** MAIN ROUTINE *********************************************
93 
 
 94 
 
94 
echo "SvarDOS repository index generator ver {$PVER}\n";
 95 
//echo "SvarDOS repository index generator ver {$PVER}\n";
95 
 
 96 
 
96 
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
 97 
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
97 
  echo "usage: php buildidx.php repodir\n";
 98 
  echo "usage: php buildidx.php repodir\n";
98 
  exit(1);
 99 
  exit(1);
99 
}
 100 
}
100 
 
 101 
 
101 
$repodir = $_SERVER['argv'][1];
 102 
$repodir = $_SERVER['argv'][1];
102 
 
 103 
 
103 
echo "building index for directory {$repodir}...\n";
 - 
 
104 
 
 - 
 
105 
$pkgfiles = scandir($repodir);
 104 
$pkgfiles = scandir($repodir);
106 
$pkglist = '';
 - 
 
107 
$pkgcount = 0;
 105 
$pkgcount = 0;
108 
 
 106 
 
109 
// iterate over each svp package
 - 
 
110 
foreach ($pkgfiles as $zipfile) {
 - 
 
111 
  if (!preg_match('/.svp$/i', $zipfile)) continue; // skip non-svp files
 107 
// do a list of all svp packages with their available versions and descriptions
112 
  if (strchr($zipfile, '-')) {
 - 
 
113 
    echo "skipping: {$zipfile}\n";
 - 
 
114 
    continue; // skip alt vers (like dosmid-0.9.2.svp)
 - 
 
115 
  }
 - 
 
116 
 
 108 
 
117 
  $path_parts = pathinfo($zipfile);
 109 
$pkgdb = array();
118 
  $pkg = strtolower($path_parts['filename']);
 110 
foreach ($pkgfiles as $fname) {
- 
 
 111 
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
119 
 
 112 
 
- 
 
 113 
  $path_parts = pathinfo($fname);
- 
 
 114 
  $pkgnam = explode('-', $path_parts['filename'])[0];
120 
  $zipfile_fullpath = realpath($repodir . '/' . $zipfile);
 115 
  $pkgfullpath = realpath($repodir . '/' . $fname);
121 
 
 116 
 
122 
  $lsm = read_file_from_zip($zipfile_fullpath, "appinfo/{$pkg}.lsm");
 117 
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
123 
  if ($lsm == false) {
 118 
  if ($lsm == false) {
124 
    echo "ERROR: pkg {$z} does not contain an LSM file at the expected location\n";
 119 
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
125 
    exit(1);
 120 
    continue;
126 
  }
 121 
  }
127 
 
 - 
 
128 
  $lsmarray = parse_lsm($lsm);
 122 
  $lsmarray = parse_lsm($lsm);
129 
  if (empty($lsmarray['version'])) {
 123 
  if (empty($lsmarray['version'])) {
130 
    echo "ERROR: lsm file in {$zipfile} does not contain a version\n";
 124 
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
131 
    var_dump($lsmarray);
 - 
 
132 
    exit(1);
 125 
    continue;
133 
  }
 126 
  }
134 
  if (empty($lsmarray['description'])) {
 127 
  if (empty($lsmarray['description'])) {
135 
    echo "ERROR: lsm file in {$zipfile} does not contain a description\n";
 128 
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
136 
    exit(1);
 129 
    continue;
- 
 
 130 
  }
- 
 
 131 
 
- 
 
 132 
  $meta['fname'] = $fname;
- 
 
 133 
  $meta['desc'] = $lsmarray['description'];
- 
 
 134 
 
- 
 
 135 
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
- 
 
 136 
}
- 
 
 137 
 
- 
 
 138 
$db = array();
- 
 
 139 
 
- 
 
 140 
// iterate over each svp package
- 
 
 141 
foreach ($pkgdb as $pkg => $versions) {
- 
 
 142 
 
- 
 
 143 
  // sort filenames by version, highest first
- 
 
 144 
  uksort($versions, "version_compare");
- 
 
 145 
  $versions = array_reverse($versions, true);
- 
 
 146 
 
- 
 
 147 
  foreach ($versions as $ver => $meta) {
- 
 
 148 
    $fname = $meta['fname'];
- 
 
 149 
    $desc = $meta['desc'];
- 
 
 150 
 
- 
 
 151 
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
- 
 
 152 
 
- 
 
 153 
    $meta2['ver'] = strval($ver);
- 
 
 154 
    $meta2['bsum'] = $bsum;
- 
 
 155 
 
- 
 
 156 
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
- 
 
 157 
    $db[$pkg]['versions'][$fname] = $meta2;
137 
  }
 158 
  }
138 
 
 159 
 
139 
  $pkglist .= "{$pkg}\t{$lsmarray['version']}\t{$lsmarray['description']}\t" . file2bsum($zipfile_fullpath) . "\n";
 - 
 
140 
  $pkgcount++;
 160 
  $pkgcount++;
141 
 
 161 
 
142 
}
 162 
}
143 
 
 163 
 
144 
echo "DONE - processed " . $pkgcount . " svp packages\n";
 164 
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
145 
 
 165 
 
146 
file_put_contents($repodir . '/index.tsv', $pkglist);
 166 
file_put_contents($repodir . '/_index.json', json_encode($db));
147 
 
 167 
 
148 
exit(0);
 168 
exit(0);
149 
 
 169 
 
150 
?>
 170 
?>
151 
 
 171