Subversion Repositories SvarDOS

Rev

Rev 923 | Rev 926 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
562 mateuszvis 1
<?php /*
2
 
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
734 bttr 6
  buildidx computes an index json file for the SvarDOS repository.
673 mateusz.vi 7
  it must be executed pointing to a directory that stores packages (*.svp)
562 mateuszvis 8
  files. buildidx will generate the index file and save it into the package
9
  repository.
10
 
11
  requires php-zip
12
 
912 mateusz.vi 13
  21 feb 2022: buildidx collects categories looking at the dir layout of each package + improved version string parsing (replaced version_compare call by dos_version_compare)
775 mateusz.vi 14
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
736 mateusz.vi 15
  16 feb 2022: added warning about overlong version strings and wild files location
719 mateusz.vi 16
  15 feb 2022: index is generated as json, contains all filenames and alt versions
673 mateusz.vi 17
  14 feb 2022: packages are expected to have the *.svp extension
650 mateusz.vi 18
  12 feb 2022: skip source packages from being processed (*.src.zip)
562 mateuszvis 19
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
20
  13 feb 2021: 'title' LSM field is no longer looked after
21
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
22
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
23
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
24
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
25
  27 aug 2016: accepting full paths to repos (starting with /...)
26
  07 dec 2013: rewritten buildidx in ANSI C89
27
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
28
  22 jul 2013: creating a listing.txt file with list of packages
29
  18 jul 2013: writing the number of packaged into the first line of the lst file
30
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
31
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
32
  04 feb 2013: added CRC32 support
33
  22 sep 2012: forked 1st version from FDUPDATE builder
34
*/
35
 
921 mateusz.vi 36
$PVER = "20220222";
562 mateuszvis 37
 
38
 
39
// computes the BSD sum of a file and returns it
40
function file2bsum($fname) {
41
  $result = 0;
42
 
43
  $fd = fopen($fname, 'rb');
44
  if ($fd === false) return(0);
45
 
46
  while (!feof($fd)) {
47
 
48
    $buff = fread($fd, 1024 * 1024);
49
 
563 mateuszvis 50
    $slen = strlen($buff);
51
    for ($i = 0; $i < $slen; $i++) {
562 mateuszvis 52
      // rotr
53
      $result = ($result >> 1) | ($result << 15);
54
      // add and truncate to 16 bits
563 mateuszvis 55
      $result += ord($buff[$i]);
562 mateuszvis 56
      $result &= 0xffff;
57
    }
58
  }
59
 
60
  fclose($fd);
61
  return($result);
62
}
63
 
64
 
912 mateusz.vi 65
// translates a version string into a array of integer values.
66
// Accepted formats follow:
67
//    300.12.1
68
//    1
69
//    12.2.34.2-4.5
70
//    1.2c
71
//    1.01 beta+3
72
//    2013-12-31
73
//    20220222 alpha
74
function vertoarr($verstr) {
75
  $subver = array(0,0,0,0);
76
 
77
  // switch string to lcase for easier processing and trim any leading or trailing white spaces
78
  $verstr = strtolower(trim($verstr));
79
 
80
  // replace all '-' and '/' characters to '.' (uniformization of sub-version parts delimiters)
81
  $verstr = strtr($verstr, '-/', '..');
82
 
83
  // is there a subversion value? (for example "+4" in "1.05+4")
84
  $i = strrpos($verstr, '+', 1);
85
  if ($i !== false) {
86
    // validate the svar-version is a proper integer
87
    $svarver = substr($verstr, $i + 1);
88
    if (! preg_match('/[1-9][0-9]*/', $svarver)) {
89
      return(false);
90
    }
91
    $subver[3] = intval($svarver); // set the +rev as a very minor item
92
    $verstr = substr($verstr, 0, $i);
93
  }
94
 
921 mateusz.vi 95
  // is the version ending with ' alpha', 'beta', etc?
922 mateusz.vi 96
  if (preg_match('/ (alpha|beta|gamma|delta|pre|rc|patch)( [0-9]{1,4}){0,1}$/', $verstr)) {
921 mateusz.vi 97
    // if there is a trailing beta-number, process it first
98
    if (preg_match('/ [0-9]{1,4}$/', $verstr)) {
99
      $i = strrpos($verstr, ' ');
100
      $subver[2] = intval(substr($verstr, $i + 1));
101
      $verstr = trim(substr($verstr, 0, $i));
102
    }
912 mateusz.vi 103
    $i = strrpos($verstr, ' ');
104
    $greek = substr($verstr, $i + 1);
105
    $verstr = trim(substr($verstr, 0, $i));
106
    if ($greek == 'alpha') {
921 mateusz.vi 107
      $subver[1] = 1;
912 mateusz.vi 108
    } else if ($greek == 'beta') {
921 mateusz.vi 109
      $subver[1] = 2;
920 mateusz.vi 110
    } else if ($greek == 'gamma') {
921 mateusz.vi 111
      $subver[1] = 3;
920 mateusz.vi 112
    } else if ($greek == 'delta') {
921 mateusz.vi 113
      $subver[1] = 4;
114
    } else if ($greek == 'pre') {
115
      $subver[1] = 5;
914 mateusz.vi 116
    } else if ($greek == 'rc') {
921 mateusz.vi 117
      $subver[1] = 6;
922 mateusz.vi 118
    } else if ($greek == 'patch') { // this is a POST-release version, as opposed to all above that are PRE-release versions
119
      $subver[1] = 99;
912 mateusz.vi 120
    } else {
121
      return(false);
122
    }
914 mateusz.vi 123
  } else {
922 mateusz.vi 124
    $subver[1] = 98; // one less than the 'patch' level
912 mateusz.vi 125
  }
126
 
127
  // does the version string have a single-letter subversion? (1.0c)
128
  if (preg_match('/[a-z]$/', $verstr)) {
921 mateusz.vi 129
    $subver[0] = ord(substr($verstr, -1));
912 mateusz.vi 130
    $verstr = substr_replace($verstr, '', -1); // remove last character from string
131
  }
132
 
923 mateusz.vi 133
  // convert "30-jan-99" and "30-jan-1999" versions to "30jan99" and "30jan1999"
925 mateusz.vi 134
  if (preg_match('/^[0-3][0-9].(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec).([0-9][0-9]){1,2}$/', $verstr)) {
923 mateusz.vi 135
    $dy = substr($verstr, 0, 2);
136
    $mo = substr($verstr, 3, 3);
137
    $ye = substr($verstr, 7);
925 mateusz.vi 138
    $verstr = "{$dy}{$mo}{$ye}";
923 mateusz.vi 139
  }
140
 
141
  // convert "30jan99" versions to 99.1.30 and "30jan1999" to 1999.1.30
925 mateusz.vi 142
  if (preg_match('/^[0-3][0-9](jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)([0-9][0-9]){1,2}$/', $verstr)) {
923 mateusz.vi 143
    $months = array('jan' => 1, 'feb' => 2, 'mar' => 3, 'apr' => 4, 'may' => 5, 'jun' => 6, 'jul' => 7, 'aug' => 8, 'sep' => 9, 'oct' => 10, 'nov' => 11, 'dec' => 12);
144
    $dy = substr($verstr, 0, 2);
145
    $mo = $months[substr($verstr, 2, 3)];
146
    $ye = substr($verstr, 5);
147
    $verstr = "{$ye}.{$mo}.{$dy}";
148
  }
149
 
912 mateusz.vi 150
  // validate the format is supported, should be something no more complex than 1.05.3.33
919 mateusz.vi 151
  if (! preg_match('/^[0-9][0-9.]{0,20}$/', $verstr)) {
912 mateusz.vi 152
    return(false);
153
  }
154
 
155
  // NOTE: a zero right after a separator and trailed with a digit (as in 1.01)
156
  //       has a special meaning
157
  $exploded = explode('.', $verstr);
158
  if (count($exploded) > 16) {
159
    return(false);
160
  }
921 mateusz.vi 161
  $exploded[16] = $subver[0]; // a-z (1.0c)
162
  $exploded[17] = $subver[1]; // alpha/beta/gamma/delta/rc/pre
163
  $exploded[18] = $subver[2]; // alpha-beta-gamma subversion (eg. "beta 9")
912 mateusz.vi 164
  $exploded[19] = $subver[3]; // svar-ver (1.0+5)
165
  for ($i = 0; $i < 20; $i++) if (empty($exploded[$i])) $exploded[$i] = '0';
166
 
167
  ksort($exploded);
168
 
169
  return($exploded);
170
}
171
 
172
 
173
function dos_version_compare($v1, $v2) {
174
  $v1arr = vertoarr($v1);
175
  $v2arr = vertoarr($v2);
176
  for ($i = 0; $i < count($v1arr); $i++) {
921 mateusz.vi 177
    if ($v1arr[$i] > $v2arr[$i]) return(1);
178
    if ($v1arr[$i] < $v2arr[$i]) return(-1);
912 mateusz.vi 179
  }
180
  return(0);
181
}
182
 
183
 
562 mateuszvis 184
// reads file fil from zip archive z and returns its content, or false on error
185
function read_file_from_zip($z, $fil) {
186
  $zip = new ZipArchive;
187
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
188
    echo "ERROR: failed to open zip file '{$z}'\n";
189
    return(false);
190
  }
191
 
192
  // load the appinfo/pkgname.lsm file
193
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
194
 
195
  $zip->close();
196
  return($res);
197
}
198
 
199
 
731 mateusz.vi 200
function read_list_of_files_in_zip($z) {
201
  $zip = new ZipArchive;
202
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
203
    echo "ERROR: failed to open zip file '{$z}'\n";
204
    return(false);
205
  }
206
 
207
  $res = array();
208
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
209
 
210
  $zip->close();
211
  return($res);
212
}
213
 
214
 
562 mateuszvis 215
// reads a LSM string and returns it in the form of an array
216
function parse_lsm($s) {
217
  $res = array();
218
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
219
    // the line is "token: value", let's find the colon
220
    $colpos = strpos($l, ':');
221
    if (($colpos === false) || ($colpos === 0)) continue;
222
    $tok = strtolower(trim(substr($l, 0, $colpos)));
223
    $val = trim(substr($l, $colpos + 1));
224
    $res[$tok] = $val;
225
  }
226
  return($res);
227
}
228
 
229
 
731 mateusz.vi 230
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
231
function str_head_is($haystack, $needle) {
232
  return strpos($haystack, $needle) === 0;
233
}
234
 
235
 
791 mateusz.vi 236
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
237
function load_core_list($repodir) {
238
  $res = array();
239
 
240
  foreach (scandir($repodir . '/core/') as $f) {
241
    if (!preg_match('/\.svp$/', $f)) continue;
242
    $res[] = explode('.', $f)[0];
243
  }
244
  return($res);
245
}
246
 
247
 
562 mateuszvis 248
// ***************** MAIN ROUTINE *********************************************
249
 
719 mateusz.vi 250
//echo "SvarDOS repository index generator ver {$PVER}\n";
562 mateuszvis 251
 
252
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
253
  echo "usage: php buildidx.php repodir\n";
254
  exit(1);
255
}
256
 
257
$repodir = $_SERVER['argv'][1];
258
 
259
$pkgfiles = scandir($repodir);
260
$pkgcount = 0;
261
 
738 mateusz.vi 262
 
795 mateusz.vi 263
// load the list of CORE and MSDOS_COMPAT packages
738 mateusz.vi 264
 
791 mateusz.vi 265
$core_packages_list = load_core_list($repodir);
804 bttr 266
$msdos_compat_list = explode(' ', 'append assign attrib chkdsk choice command comp cpidos debug defrag deltree diskcomp diskcopy display edit edlin exe2bin fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mirror mode more move nlsfunc print replace share shsucdx sort swsubst tree undelete unformat xcopy');
738 mateusz.vi 267
 
719 mateusz.vi 268
// do a list of all svp packages with their available versions and descriptions
562 mateuszvis 269
 
719 mateusz.vi 270
$pkgdb = array();
271
foreach ($pkgfiles as $fname) {
801 mateusz.vi 272
  if (!preg_match('/\.svp$/i', $fname)) continue; // skip non-svp files
562 mateuszvis 273
 
801 mateusz.vi 274
  if (!preg_match('/^[a-zA-Z0-9+. _-]*\.svp$/', $fname)) {
275
    echo "ERROR: {$fname} has a very weird name\n";
276
    continue;
277
  }
278
 
719 mateusz.vi 279
  $path_parts = pathinfo($fname);
280
  $pkgnam = explode('-', $path_parts['filename'])[0];
281
  $pkgfullpath = realpath($repodir . '/' . $fname);
562 mateuszvis 282
 
719 mateusz.vi 283
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
562 mateuszvis 284
  if ($lsm == false) {
802 mateusz.vi 285
    echo "ERROR: {$fname} does not contain an LSM file at the expected location\n";
719 mateusz.vi 286
    continue;
562 mateuszvis 287
  }
288
  $lsmarray = parse_lsm($lsm);
289
  if (empty($lsmarray['version'])) {
719 mateusz.vi 290
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
291
    continue;
562 mateuszvis 292
  }
730 mateusz.vi 293
  if (strlen($lsmarray['version']) > 16) {
737 mateusz.vi 294
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
730 mateusz.vi 295
    continue;
296
  }
562 mateuszvis 297
  if (empty($lsmarray['description'])) {
719 mateusz.vi 298
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
299
    continue;
562 mateuszvis 300
  }
301
 
731 mateusz.vi 302
  // validate the files present in the archive
303
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
739 mateusz.vi 304
  $pkgdir = $pkgnam;
305
 
768 mateusz.vi 306
  // special rule for "parent and children" packages
307
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
754 mateusz.vi 308
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
802 mateusz.vi 309
  if ($pkgnam == 'clamdb') $pkgdir = 'clamav'; // data patterns for clamav
739 mateusz.vi 310
 
768 mateusz.vi 311
  // array used to detect duplicated entries after lower-case conversion
312
  $duparr = array();
313
 
909 mateusz.vi 314
  // will hold the list of categories that this package belongs to
315
  $catlist = array();
316
 
731 mateusz.vi 317
  foreach ($listoffiles as $f) {
318
    $f = strtolower($f);
768 mateusz.vi 319
    $path_array = explode('/', $f);
320
    // emit a warning when non-8+3 filenames are spotted and find duplicates
321
    foreach ($path_array as $item) {
322
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
323
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
324
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
325
      }
326
    }
327
    // look for dups
328
    if (array_search($f, $duparr) !== false) {
329
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
330
    } else {
331
      $duparr[] = $f;
332
    }
731 mateusz.vi 333
    // LSM file is ok
334
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
335
    if ($f === "appinfo/") continue;
795 mateusz.vi 336
    // CORE and MSDOS_COMPAT packages are premium citizens and can do a little more
909 mateusz.vi 337
    $core_or_msdoscompat = 0;
338
    if (array_search($pkgnam, $core_packages_list) !== false) {
339
      $catlist[] = 'core';
340
      $core_or_msdoscompat = 1;
341
    }
342
    if (array_search($pkgnam, $msdos_compat_list) !== false) {
343
      $catlist[] = 'msdos_compat';
344
      $core_or_msdoscompat = 1;
345
    }
346
    if ($core_or_msdoscompat == 1) {
736 mateusz.vi 347
      if (str_head_is($f, 'bin/')) continue;
779 mateusz.vi 348
      if (str_head_is($f, 'cpi/')) continue;
749 mateusz.vi 349
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
350
      if ($f === 'doc/') continue;
351
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
352
      if ($f === 'nls/') continue;
736 mateusz.vi 353
    }
798 mateusz.vi 354
    // the help package is allowed to put files in... help
355
    if (($pkgnam == 'help') && (str_head_is($f, 'help/'))) continue;
909 mateusz.vi 356
    // must be category-prefixed file, add it to the list of categories for this package
357
    $catlist[] = explode('/', $f)[0];
749 mateusz.vi 358
    // well-known "category" dirs are okay
739 mateusz.vi 359
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
731 mateusz.vi 360
    if ($f === 'progs/') continue;
739 mateusz.vi 361
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
731 mateusz.vi 362
    if ($f === 'devel/') continue;
739 mateusz.vi 363
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
731 mateusz.vi 364
    if ($f === 'games/') continue;
739 mateusz.vi 365
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
731 mateusz.vi 366
    if ($f === 'drivers/') continue;
768 mateusz.vi 367
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
731 mateusz.vi 368
  }
369
 
912 mateusz.vi 370
  // do I understand the version string?
371
  if (vertoarr($lsmarray['version']) === false) echo "WARNING: {$fname} parsing of version string failed ('{$lsmarray['version']}')\n";
372
 
719 mateusz.vi 373
  $meta['fname'] = $fname;
374
  $meta['desc'] = $lsmarray['description'];
909 mateusz.vi 375
  $meta['cats'] = array_unique($catlist);
719 mateusz.vi 376
 
377
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
378
}
379
 
801 mateusz.vi 380
 
719 mateusz.vi 381
$db = array();
909 mateusz.vi 382
$cats = array();
719 mateusz.vi 383
 
909 mateusz.vi 384
// ******** compute the version-sorted list of packages with a single *********
385
// ******** description and category list for each package ********************
386
 
719 mateusz.vi 387
// iterate over each svp package
388
foreach ($pkgdb as $pkg => $versions) {
389
 
390
  // sort filenames by version, highest first
912 mateusz.vi 391
  uksort($versions, "dos_version_compare");
719 mateusz.vi 392
  $versions = array_reverse($versions, true);
393
 
394
  foreach ($versions as $ver => $meta) {
395
    $fname = $meta['fname'];
396
    $desc = $meta['desc'];
397
 
398
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
399
 
400
    $meta2['ver'] = strval($ver);
401
    $meta2['bsum'] = $bsum;
402
 
403
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
909 mateusz.vi 404
    if (empty($db[$pkg]['cats'])) {
405
      $db[$pkg]['cats'] = $meta['cats'];
406
      $cats = array_unique(array_merge($cats, $meta['cats']));
407
    }
719 mateusz.vi 408
    $db[$pkg]['versions'][$fname] = $meta2;
409
  }
410
 
562 mateuszvis 411
  $pkgcount++;
412
 
413
}
414
 
719 mateusz.vi 415
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
562 mateuszvis 416
 
801 mateusz.vi 417
$json_blob = json_encode($db);
418
if ($json_blob === false) {
419
  echo "ERROR: JSON convertion failed! -> ";
420
  switch (json_last_error()) {
421
    case JSON_ERROR_DEPTH:
422
      echo 'maximum stack depth exceeded';
423
      break;
424
    case JSON_ERROR_STATE_MISMATCH:
425
      echo 'underflow of the modes mismatch';
426
      break;
427
    case JSON_ERROR_CTRL_CHAR:
428
      echo 'unexpected control character found';
429
      break;
430
    case JSON_ERROR_UTF8:
431
      echo 'malformed utf-8 characters';
432
      break;
433
    default:
434
      echo "unknown error";
435
      break;
436
  }
437
  echo "\n";
438
}
439
 
909 mateusz.vi 440
file_put_contents($repodir . '/_index.json', $json_blob);
562 mateuszvis 441
 
909 mateusz.vi 442
$cats_json = json_encode($cats);
443
file_put_contents($repodir . '/_cats.json', $cats_json);
444
 
562 mateuszvis 445
exit(0);
446
 
447
?>