Subversion Repositories SvarDOS

Rev

Rev 779 | Rev 791 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 779 Rev 781
1
<?php /*
1
<?php /*
2
 
2
 
3
  SvarDOS repo index builder
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
5
 
6
  buildidx computes an index json file for the SvarDOS repository.
6
  buildidx computes an index json file for the SvarDOS repository.
7
  it must be executed pointing to a directory that stores packages (*.svp)
7
  it must be executed pointing to a directory that stores packages (*.svp)
8
  files. buildidx will generate the index file and save it into the package
8
  files. buildidx will generate the index file and save it into the package
9
  repository.
9
  repository.
10
 
10
 
11
  requires php-zip
11
  requires php-zip
12
 
12
 
13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
14
  16 feb 2022: added warning about overlong version strings and wild files location
14
  16 feb 2022: added warning about overlong version strings and wild files location
15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
16
  14 feb 2022: packages are expected to have the *.svp extension
16
  14 feb 2022: packages are expected to have the *.svp extension
17
  12 feb 2022: skip source packages from being processed (*.src.zip)
17
  12 feb 2022: skip source packages from being processed (*.src.zip)
18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
19
  13 feb 2021: 'title' LSM field is no longer looked after
19
  13 feb 2021: 'title' LSM field is no longer looked after
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
25
  07 dec 2013: rewritten buildidx in ANSI C89
25
  07 dec 2013: rewritten buildidx in ANSI C89
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
27
  22 jul 2013: creating a listing.txt file with list of packages
27
  22 jul 2013: creating a listing.txt file with list of packages
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
31
  04 feb 2013: added CRC32 support
31
  04 feb 2013: added CRC32 support
32
  22 sep 2012: forked 1st version from FDUPDATE builder
32
  22 sep 2012: forked 1st version from FDUPDATE builder
33
*/
33
*/
34
 
34
 
35
$PVER = "20220216";
35
$PVER = "20220216";
36
 
36
 
37
 
37
 
38
// computes the BSD sum of a file and returns it
38
// computes the BSD sum of a file and returns it
39
function file2bsum($fname) {
39
function file2bsum($fname) {
40
  $result = 0;
40
  $result = 0;
41
 
41
 
42
  $fd = fopen($fname, 'rb');
42
  $fd = fopen($fname, 'rb');
43
  if ($fd === false) return(0);
43
  if ($fd === false) return(0);
44
 
44
 
45
  while (!feof($fd)) {
45
  while (!feof($fd)) {
46
 
46
 
47
    $buff = fread($fd, 1024 * 1024);
47
    $buff = fread($fd, 1024 * 1024);
48
 
48
 
49
    $slen = strlen($buff);
49
    $slen = strlen($buff);
50
    for ($i = 0; $i < $slen; $i++) {
50
    for ($i = 0; $i < $slen; $i++) {
51
      // rotr
51
      // rotr
52
      $result = ($result >> 1) | ($result << 15);
52
      $result = ($result >> 1) | ($result << 15);
53
      // add and truncate to 16 bits
53
      // add and truncate to 16 bits
54
      $result += ord($buff[$i]);
54
      $result += ord($buff[$i]);
55
      $result &= 0xffff;
55
      $result &= 0xffff;
56
    }
56
    }
57
  }
57
  }
58
 
58
 
59
  fclose($fd);
59
  fclose($fd);
60
  return($result);
60
  return($result);
61
}
61
}
62
 
62
 
63
 
63
 
64
// reads file fil from zip archive z and returns its content, or false on error
64
// reads file fil from zip archive z and returns its content, or false on error
65
function read_file_from_zip($z, $fil) {
65
function read_file_from_zip($z, $fil) {
66
  $zip = new ZipArchive;
66
  $zip = new ZipArchive;
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
68
    echo "ERROR: failed to open zip file '{$z}'\n";
68
    echo "ERROR: failed to open zip file '{$z}'\n";
69
    return(false);
69
    return(false);
70
  }
70
  }
71
 
71
 
72
  // load the appinfo/pkgname.lsm file
72
  // load the appinfo/pkgname.lsm file
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
74
 
74
 
75
  $zip->close();
75
  $zip->close();
76
  return($res);
76
  return($res);
77
}
77
}
78
 
78
 
79
 
79
 
80
function read_list_of_files_in_zip($z) {
80
function read_list_of_files_in_zip($z) {
81
  $zip = new ZipArchive;
81
  $zip = new ZipArchive;
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
83
    echo "ERROR: failed to open zip file '{$z}'\n";
83
    echo "ERROR: failed to open zip file '{$z}'\n";
84
    return(false);
84
    return(false);
85
  }
85
  }
86
 
86
 
87
  $res = array();
87
  $res = array();
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
89
 
89
 
90
  $zip->close();
90
  $zip->close();
91
  return($res);
91
  return($res);
92
}
92
}
93
 
93
 
94
 
94
 
95
// reads a LSM string and returns it in the form of an array
95
// reads a LSM string and returns it in the form of an array
96
function parse_lsm($s) {
96
function parse_lsm($s) {
97
  $res = array();
97
  $res = array();
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
99
    // the line is "token: value", let's find the colon
99
    // the line is "token: value", let's find the colon
100
    $colpos = strpos($l, ':');
100
    $colpos = strpos($l, ':');
101
    if (($colpos === false) || ($colpos === 0)) continue;
101
    if (($colpos === false) || ($colpos === 0)) continue;
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
103
    $val = trim(substr($l, $colpos + 1));
103
    $val = trim(substr($l, $colpos + 1));
104
    $res[$tok] = $val;
104
    $res[$tok] = $val;
105
  }
105
  }
106
  return($res);
106
  return($res);
107
}
107
}
108
 
108
 
109
 
109
 
110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
111
function str_head_is($haystack, $needle) {
111
function str_head_is($haystack, $needle) {
112
  return strpos($haystack, $needle) === 0;
112
  return strpos($haystack, $needle) === 0;
113
}
113
}
114
 
114
 
115
 
115
 
116
// ***************** MAIN ROUTINE *********************************************
116
// ***************** MAIN ROUTINE *********************************************
117
 
117
 
118
//echo "SvarDOS repository index generator ver {$PVER}\n";
118
//echo "SvarDOS repository index generator ver {$PVER}\n";
119
 
119
 
120
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
120
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
121
  echo "usage: php buildidx.php repodir\n";
121
  echo "usage: php buildidx.php repodir\n";
122
  exit(1);
122
  exit(1);
123
}
123
}
124
 
124
 
125
$repodir = $_SERVER['argv'][1];
125
$repodir = $_SERVER['argv'][1];
126
 
126
 
127
$pkgfiles = scandir($repodir);
127
$pkgfiles = scandir($repodir);
128
$pkgcount = 0;
128
$pkgcount = 0;
129
 
129
 
130
 
130
 
131
// load the list of CORE packages
131
// load the list of CORE packages
132
 
132
 
133
$core_packages_list = explode(' ', 'amb attrib chkdsk choice command cpidos debug deltree diskcopy display dosfsck edit fc fdapm fdisk find format help himemx kernel keyb keyb_lay label localcfg mem mode more move pkg pkgnet shsucdx sort tree');
133
$core_packages_list = explode(' ', 'amb attrib chkdsk choice command cpidos debug deltree diskcopy display dosfsck edit fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mode more move pkg pkgnet shsucdx sort tree');
134
 
134
 
135
 
135
 
136
// do a list of all svp packages with their available versions and descriptions
136
// do a list of all svp packages with their available versions and descriptions
137
 
137
 
138
$pkgdb = array();
138
$pkgdb = array();
139
foreach ($pkgfiles as $fname) {
139
foreach ($pkgfiles as $fname) {
140
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
140
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
141
 
141
 
142
  $path_parts = pathinfo($fname);
142
  $path_parts = pathinfo($fname);
143
  $pkgnam = explode('-', $path_parts['filename'])[0];
143
  $pkgnam = explode('-', $path_parts['filename'])[0];
144
  $pkgfullpath = realpath($repodir . '/' . $fname);
144
  $pkgfullpath = realpath($repodir . '/' . $fname);
145
 
145
 
146
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
146
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
147
  if ($lsm == false) {
147
  if ($lsm == false) {
148
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
148
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
149
    continue;
149
    continue;
150
  }
150
  }
151
  $lsmarray = parse_lsm($lsm);
151
  $lsmarray = parse_lsm($lsm);
152
  if (empty($lsmarray['version'])) {
152
  if (empty($lsmarray['version'])) {
153
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
153
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
154
    continue;
154
    continue;
155
  }
155
  }
156
  if (strlen($lsmarray['version']) > 16) {
156
  if (strlen($lsmarray['version']) > 16) {
157
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
157
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
158
    continue;
158
    continue;
159
  }
159
  }
160
  if (empty($lsmarray['description'])) {
160
  if (empty($lsmarray['description'])) {
161
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
161
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
162
    continue;
162
    continue;
163
  }
163
  }
164
 
164
 
165
  // validate the files present in the archive
165
  // validate the files present in the archive
166
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
166
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
167
  $pkgdir = $pkgnam;
167
  $pkgdir = $pkgnam;
168
 
168
 
169
  // special rule for "parent and children" packages
169
  // special rule for "parent and children" packages
170
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
170
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
171
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
171
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
172
 
172
 
173
  // array used to detect duplicated entries after lower-case conversion
173
  // array used to detect duplicated entries after lower-case conversion
174
  $duparr = array();
174
  $duparr = array();
175
 
175
 
176
  foreach ($listoffiles as $f) {
176
  foreach ($listoffiles as $f) {
177
    $f = strtolower($f);
177
    $f = strtolower($f);
178
    $path_array = explode('/', $f);
178
    $path_array = explode('/', $f);
179
    // emit a warning when non-8+3 filenames are spotted and find duplicates
179
    // emit a warning when non-8+3 filenames are spotted and find duplicates
180
    foreach ($path_array as $item) {
180
    foreach ($path_array as $item) {
181
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
181
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
182
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
182
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
183
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
183
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
184
      }
184
      }
185
    }
185
    }
186
    // look for dups
186
    // look for dups
187
    if (array_search($f, $duparr) !== false) {
187
    if (array_search($f, $duparr) !== false) {
188
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
188
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
189
    } else {
189
    } else {
190
      $duparr[] = $f;
190
      $duparr[] = $f;
191
    }
191
    }
192
    // LSM file is ok
192
    // LSM file is ok
193
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
193
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
194
    if ($f === "appinfo/") continue;
194
    if ($f === "appinfo/") continue;
195
    // CORE packages are premium citizens and can do a little more
195
    // CORE packages are premium citizens and can do a little more
196
    if (array_search($pkgnam, $core_packages_list) !== false) {
196
    if (array_search($pkgnam, $core_packages_list) !== false) {
197
      if (str_head_is($f, 'bin/')) continue;
197
      if (str_head_is($f, 'bin/')) continue;
198
      if (str_head_is($f, 'cpi/')) continue;
198
      if (str_head_is($f, 'cpi/')) continue;
199
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
199
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
200
      if ($f === 'doc/') continue;
200
      if ($f === 'doc/') continue;
201
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
201
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
202
      if ($f === 'nls/') continue;
202
      if ($f === 'nls/') continue;
203
    }
203
    }
204
    // well-known "category" dirs are okay
204
    // well-known "category" dirs are okay
205
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
205
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
206
    if ($f === 'progs/') continue;
206
    if ($f === 'progs/') continue;
207
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
207
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
208
    if ($f === 'devel/') continue;
208
    if ($f === 'devel/') continue;
209
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
209
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
210
    if ($f === 'games/') continue;
210
    if ($f === 'games/') continue;
211
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
211
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
212
    if ($f === 'drivers/') continue;
212
    if ($f === 'drivers/') continue;
213
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
213
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
214
  }
214
  }
215
 
215
 
216
  $meta['fname'] = $fname;
216
  $meta['fname'] = $fname;
217
  $meta['desc'] = $lsmarray['description'];
217
  $meta['desc'] = $lsmarray['description'];
218
 
218
 
219
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
219
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
220
}
220
}
221
 
221
 
222
$db = array();
222
$db = array();
223
 
223
 
224
// iterate over each svp package
224
// iterate over each svp package
225
foreach ($pkgdb as $pkg => $versions) {
225
foreach ($pkgdb as $pkg => $versions) {
226
 
226
 
227
  // sort filenames by version, highest first
227
  // sort filenames by version, highest first
228
  uksort($versions, "version_compare");
228
  uksort($versions, "version_compare");
229
  $versions = array_reverse($versions, true);
229
  $versions = array_reverse($versions, true);
230
 
230
 
231
  foreach ($versions as $ver => $meta) {
231
  foreach ($versions as $ver => $meta) {
232
    $fname = $meta['fname'];
232
    $fname = $meta['fname'];
233
    $desc = $meta['desc'];
233
    $desc = $meta['desc'];
234
 
234
 
235
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
235
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
236
 
236
 
237
    $meta2['ver'] = strval($ver);
237
    $meta2['ver'] = strval($ver);
238
    $meta2['bsum'] = $bsum;
238
    $meta2['bsum'] = $bsum;
239
 
239
 
240
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
240
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
241
    $db[$pkg]['versions'][$fname] = $meta2;
241
    $db[$pkg]['versions'][$fname] = $meta2;
242
  }
242
  }
243
 
243
 
244
  $pkgcount++;
244
  $pkgcount++;
245
 
245
 
246
}
246
}
247
 
247
 
248
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
248
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
249
 
249
 
250
file_put_contents($repodir . '/_index.json', json_encode($db));
250
file_put_contents($repodir . '/_index.json', json_encode($db));
251
 
251
 
252
exit(0);
252
exit(0);
253
 
253
 
254
?>
254
?>
255
 
255