Subversion Repositories SvarDOS

Rev

Rev 781 | Rev 795 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
562 mateuszvis 1
<?php /*
2
 
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
734 bttr 6
  buildidx computes an index json file for the SvarDOS repository.
673 mateusz.vi 7
  it must be executed pointing to a directory that stores packages (*.svp)
562 mateuszvis 8
  files. buildidx will generate the index file and save it into the package
9
  repository.
10
 
11
  requires php-zip
12
 
775 mateusz.vi 13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
736 mateusz.vi 14
  16 feb 2022: added warning about overlong version strings and wild files location
719 mateusz.vi 15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
673 mateusz.vi 16
  14 feb 2022: packages are expected to have the *.svp extension
650 mateusz.vi 17
  12 feb 2022: skip source packages from being processed (*.src.zip)
562 mateuszvis 18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
19
  13 feb 2021: 'title' LSM field is no longer looked after
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
25
  07 dec 2013: rewritten buildidx in ANSI C89
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
27
  22 jul 2013: creating a listing.txt file with list of packages
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
31
  04 feb 2013: added CRC32 support
32
  22 sep 2012: forked 1st version from FDUPDATE builder
33
*/
34
 
791 mateusz.vi 35
$PVER = "20220217";
562 mateuszvis 36
 
37
 
38
// computes the BSD sum of a file and returns it
39
function file2bsum($fname) {
40
  $result = 0;
41
 
42
  $fd = fopen($fname, 'rb');
43
  if ($fd === false) return(0);
44
 
45
  while (!feof($fd)) {
46
 
47
    $buff = fread($fd, 1024 * 1024);
48
 
563 mateuszvis 49
    $slen = strlen($buff);
50
    for ($i = 0; $i < $slen; $i++) {
562 mateuszvis 51
      // rotr
52
      $result = ($result >> 1) | ($result << 15);
53
      // add and truncate to 16 bits
563 mateuszvis 54
      $result += ord($buff[$i]);
562 mateuszvis 55
      $result &= 0xffff;
56
    }
57
  }
58
 
59
  fclose($fd);
60
  return($result);
61
}
62
 
63
 
64
// reads file fil from zip archive z and returns its content, or false on error
65
function read_file_from_zip($z, $fil) {
66
  $zip = new ZipArchive;
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
68
    echo "ERROR: failed to open zip file '{$z}'\n";
69
    return(false);
70
  }
71
 
72
  // load the appinfo/pkgname.lsm file
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
74
 
75
  $zip->close();
76
  return($res);
77
}
78
 
79
 
731 mateusz.vi 80
function read_list_of_files_in_zip($z) {
81
  $zip = new ZipArchive;
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
83
    echo "ERROR: failed to open zip file '{$z}'\n";
84
    return(false);
85
  }
86
 
87
  $res = array();
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
89
 
90
  $zip->close();
91
  return($res);
92
}
93
 
94
 
562 mateuszvis 95
// reads a LSM string and returns it in the form of an array
96
function parse_lsm($s) {
97
  $res = array();
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
99
    // the line is "token: value", let's find the colon
100
    $colpos = strpos($l, ':');
101
    if (($colpos === false) || ($colpos === 0)) continue;
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
103
    $val = trim(substr($l, $colpos + 1));
104
    $res[$tok] = $val;
105
  }
106
  return($res);
107
}
108
 
109
 
731 mateusz.vi 110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
111
function str_head_is($haystack, $needle) {
112
  return strpos($haystack, $needle) === 0;
113
}
114
 
115
 
791 mateusz.vi 116
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
117
function load_core_list($repodir) {
118
  $res = array();
119
 
120
  foreach (scandir($repodir . '/core/') as $f) {
121
    if (!preg_match('/\.svp$/', $f)) continue;
122
    $res[] = explode('.', $f)[0];
123
  }
124
  return($res);
125
}
126
 
127
 
562 mateuszvis 128
// ***************** MAIN ROUTINE *********************************************
129
 
719 mateusz.vi 130
//echo "SvarDOS repository index generator ver {$PVER}\n";
562 mateuszvis 131
 
132
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
133
  echo "usage: php buildidx.php repodir\n";
134
  exit(1);
135
}
136
 
137
$repodir = $_SERVER['argv'][1];
138
 
139
$pkgfiles = scandir($repodir);
140
$pkgcount = 0;
141
 
738 mateusz.vi 142
 
143
// load the list of CORE packages
144
 
791 mateusz.vi 145
$core_packages_list = load_core_list($repodir);
738 mateusz.vi 146
 
147
 
719 mateusz.vi 148
// do a list of all svp packages with their available versions and descriptions
562 mateuszvis 149
 
719 mateusz.vi 150
$pkgdb = array();
151
foreach ($pkgfiles as $fname) {
152
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
562 mateuszvis 153
 
719 mateusz.vi 154
  $path_parts = pathinfo($fname);
155
  $pkgnam = explode('-', $path_parts['filename'])[0];
156
  $pkgfullpath = realpath($repodir . '/' . $fname);
562 mateuszvis 157
 
719 mateusz.vi 158
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
562 mateuszvis 159
  if ($lsm == false) {
719 mateusz.vi 160
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
161
    continue;
562 mateuszvis 162
  }
163
  $lsmarray = parse_lsm($lsm);
164
  if (empty($lsmarray['version'])) {
719 mateusz.vi 165
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
166
    continue;
562 mateuszvis 167
  }
730 mateusz.vi 168
  if (strlen($lsmarray['version']) > 16) {
737 mateusz.vi 169
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
730 mateusz.vi 170
    continue;
171
  }
562 mateuszvis 172
  if (empty($lsmarray['description'])) {
719 mateusz.vi 173
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
174
    continue;
562 mateuszvis 175
  }
176
 
731 mateusz.vi 177
  // validate the files present in the archive
178
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
739 mateusz.vi 179
  $pkgdir = $pkgnam;
180
 
768 mateusz.vi 181
  // special rule for "parent and children" packages
182
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
754 mateusz.vi 183
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
739 mateusz.vi 184
 
768 mateusz.vi 185
  // array used to detect duplicated entries after lower-case conversion
186
  $duparr = array();
187
 
731 mateusz.vi 188
  foreach ($listoffiles as $f) {
189
    $f = strtolower($f);
768 mateusz.vi 190
    $path_array = explode('/', $f);
191
    // emit a warning when non-8+3 filenames are spotted and find duplicates
192
    foreach ($path_array as $item) {
193
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
194
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
195
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
196
      }
197
    }
198
    // look for dups
199
    if (array_search($f, $duparr) !== false) {
200
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
201
    } else {
202
      $duparr[] = $f;
203
    }
731 mateusz.vi 204
    // LSM file is ok
205
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
206
    if ($f === "appinfo/") continue;
736 mateusz.vi 207
    // CORE packages are premium citizens and can do a little more
738 mateusz.vi 208
    if (array_search($pkgnam, $core_packages_list) !== false) {
736 mateusz.vi 209
      if (str_head_is($f, 'bin/')) continue;
779 mateusz.vi 210
      if (str_head_is($f, 'cpi/')) continue;
749 mateusz.vi 211
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
212
      if ($f === 'doc/') continue;
213
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
214
      if ($f === 'nls/') continue;
736 mateusz.vi 215
    }
749 mateusz.vi 216
    // well-known "category" dirs are okay
739 mateusz.vi 217
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
731 mateusz.vi 218
    if ($f === 'progs/') continue;
739 mateusz.vi 219
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
731 mateusz.vi 220
    if ($f === 'devel/') continue;
739 mateusz.vi 221
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
731 mateusz.vi 222
    if ($f === 'games/') continue;
739 mateusz.vi 223
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
731 mateusz.vi 224
    if ($f === 'drivers/') continue;
768 mateusz.vi 225
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
731 mateusz.vi 226
  }
227
 
719 mateusz.vi 228
  $meta['fname'] = $fname;
229
  $meta['desc'] = $lsmarray['description'];
230
 
231
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
232
}
233
 
234
$db = array();
235
 
236
// iterate over each svp package
237
foreach ($pkgdb as $pkg => $versions) {
238
 
239
  // sort filenames by version, highest first
240
  uksort($versions, "version_compare");
241
  $versions = array_reverse($versions, true);
242
 
243
  foreach ($versions as $ver => $meta) {
244
    $fname = $meta['fname'];
245
    $desc = $meta['desc'];
246
 
247
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
248
 
249
    $meta2['ver'] = strval($ver);
250
    $meta2['bsum'] = $bsum;
251
 
252
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
253
    $db[$pkg]['versions'][$fname] = $meta2;
254
  }
255
 
562 mateuszvis 256
  $pkgcount++;
257
 
258
}
259
 
719 mateusz.vi 260
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
562 mateuszvis 261
 
719 mateusz.vi 262
file_put_contents($repodir . '/_index.json', json_encode($db));
562 mateuszvis 263
 
264
exit(0);
265
 
266
?>