Subversion Repositories SvarDOS

Rev

Rev 754 | Rev 775 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 754 Rev 768
1
<?php /*
1
<?php /*
2
 
2
 
3
  SvarDOS repo index builder
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
5
 
6
  buildidx computes an index json file for the SvarDOS repository.
6
  buildidx computes an index json file for the SvarDOS repository.
7
  it must be executed pointing to a directory that stores packages (*.svp)
7
  it must be executed pointing to a directory that stores packages (*.svp)
8
  files. buildidx will generate the index file and save it into the package
8
  files. buildidx will generate the index file and save it into the package
9
  repository.
9
  repository.
10
 
10
 
11
  requires php-zip
11
  requires php-zip
12
 
12
 
13
  16 feb 2022: added warning about overlong version strings and wild files location
13
  16 feb 2022: added warning about overlong version strings and wild files location
14
  15 feb 2022: index is generated as json, contains all filenames and alt versions
14
  15 feb 2022: index is generated as json, contains all filenames and alt versions
15
  14 feb 2022: packages are expected to have the *.svp extension
15
  14 feb 2022: packages are expected to have the *.svp extension
16
  12 feb 2022: skip source packages from being processed (*.src.zip)
16
  12 feb 2022: skip source packages from being processed (*.src.zip)
17
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
17
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
18
  13 feb 2021: 'title' LSM field is no longer looked after
18
  13 feb 2021: 'title' LSM field is no longer looked after
19
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
19
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
20
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
20
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
21
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
21
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
22
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
22
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
23
  27 aug 2016: accepting full paths to repos (starting with /...)
23
  27 aug 2016: accepting full paths to repos (starting with /...)
24
  07 dec 2013: rewritten buildidx in ANSI C89
24
  07 dec 2013: rewritten buildidx in ANSI C89
25
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
25
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
26
  22 jul 2013: creating a listing.txt file with list of packages
26
  22 jul 2013: creating a listing.txt file with list of packages
27
  18 jul 2013: writing the number of packaged into the first line of the lst file
27
  18 jul 2013: writing the number of packaged into the first line of the lst file
28
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
28
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
29
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
29
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
30
  04 feb 2013: added CRC32 support
30
  04 feb 2013: added CRC32 support
31
  22 sep 2012: forked 1st version from FDUPDATE builder
31
  22 sep 2012: forked 1st version from FDUPDATE builder
32
*/
32
*/
33
 
33
 
34
$PVER = "20220216";
34
$PVER = "20220216";
35
 
35
 
36
 
36
 
37
// computes the BSD sum of a file and returns it
37
// computes the BSD sum of a file and returns it
38
function file2bsum($fname) {
38
function file2bsum($fname) {
39
  $result = 0;
39
  $result = 0;
40
 
40
 
41
  $fd = fopen($fname, 'rb');
41
  $fd = fopen($fname, 'rb');
42
  if ($fd === false) return(0);
42
  if ($fd === false) return(0);
43
 
43
 
44
  while (!feof($fd)) {
44
  while (!feof($fd)) {
45
 
45
 
46
    $buff = fread($fd, 1024 * 1024);
46
    $buff = fread($fd, 1024 * 1024);
47
 
47
 
48
    $slen = strlen($buff);
48
    $slen = strlen($buff);
49
    for ($i = 0; $i < $slen; $i++) {
49
    for ($i = 0; $i < $slen; $i++) {
50
      // rotr
50
      // rotr
51
      $result = ($result >> 1) | ($result << 15);
51
      $result = ($result >> 1) | ($result << 15);
52
      // add and truncate to 16 bits
52
      // add and truncate to 16 bits
53
      $result += ord($buff[$i]);
53
      $result += ord($buff[$i]);
54
      $result &= 0xffff;
54
      $result &= 0xffff;
55
    }
55
    }
56
  }
56
  }
57
 
57
 
58
  fclose($fd);
58
  fclose($fd);
59
  return($result);
59
  return($result);
60
}
60
}
61
 
61
 
62
 
62
 
63
// reads file fil from zip archive z and returns its content, or false on error
63
// reads file fil from zip archive z and returns its content, or false on error
64
function read_file_from_zip($z, $fil) {
64
function read_file_from_zip($z, $fil) {
65
  $zip = new ZipArchive;
65
  $zip = new ZipArchive;
66
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
66
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
67
    echo "ERROR: failed to open zip file '{$z}'\n";
67
    echo "ERROR: failed to open zip file '{$z}'\n";
68
    return(false);
68
    return(false);
69
  }
69
  }
70
 
70
 
71
  // load the appinfo/pkgname.lsm file
71
  // load the appinfo/pkgname.lsm file
72
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
72
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
73
 
73
 
74
  $zip->close();
74
  $zip->close();
75
  return($res);
75
  return($res);
76
}
76
}
77
 
77
 
78
 
78
 
79
function read_list_of_files_in_zip($z) {
79
function read_list_of_files_in_zip($z) {
80
  $zip = new ZipArchive;
80
  $zip = new ZipArchive;
81
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
81
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
82
    echo "ERROR: failed to open zip file '{$z}'\n";
82
    echo "ERROR: failed to open zip file '{$z}'\n";
83
    return(false);
83
    return(false);
84
  }
84
  }
85
 
85
 
86
  $res = array();
86
  $res = array();
87
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
87
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
88
 
88
 
89
  $zip->close();
89
  $zip->close();
90
  return($res);
90
  return($res);
91
}
91
}
92
 
92
 
93
 
93
 
94
// reads a LSM string and returns it in the form of an array
94
// reads a LSM string and returns it in the form of an array
95
function parse_lsm($s) {
95
function parse_lsm($s) {
96
  $res = array();
96
  $res = array();
97
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
97
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
98
    // the line is "token: value", let's find the colon
98
    // the line is "token: value", let's find the colon
99
    $colpos = strpos($l, ':');
99
    $colpos = strpos($l, ':');
100
    if (($colpos === false) || ($colpos === 0)) continue;
100
    if (($colpos === false) || ($colpos === 0)) continue;
101
    $tok = strtolower(trim(substr($l, 0, $colpos)));
101
    $tok = strtolower(trim(substr($l, 0, $colpos)));
102
    $val = trim(substr($l, $colpos + 1));
102
    $val = trim(substr($l, $colpos + 1));
103
    $res[$tok] = $val;
103
    $res[$tok] = $val;
104
  }
104
  }
105
  return($res);
105
  return($res);
106
}
106
}
107
 
107
 
108
 
108
 
109
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
109
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
110
function str_head_is($haystack, $needle) {
110
function str_head_is($haystack, $needle) {
111
  return strpos($haystack, $needle) === 0;
111
  return strpos($haystack, $needle) === 0;
112
}
112
}
113
 
113
 
114
 
114
 
115
// ***************** MAIN ROUTINE *********************************************
115
// ***************** MAIN ROUTINE *********************************************
116
 
116
 
117
//echo "SvarDOS repository index generator ver {$PVER}\n";
117
//echo "SvarDOS repository index generator ver {$PVER}\n";
118
 
118
 
119
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
119
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
120
  echo "usage: php buildidx.php repodir\n";
120
  echo "usage: php buildidx.php repodir\n";
121
  exit(1);
121
  exit(1);
122
}
122
}
123
 
123
 
124
$repodir = $_SERVER['argv'][1];
124
$repodir = $_SERVER['argv'][1];
125
 
125
 
126
$pkgfiles = scandir($repodir);
126
$pkgfiles = scandir($repodir);
127
$pkgcount = 0;
127
$pkgcount = 0;
128
 
128
 
129
 
129
 
130
// load the list of CORE packages
130
// load the list of CORE packages
131
 
131
 
132
$core_packages_list = explode(' ', 'amb attrib chkdsk choice command cpidos debug deltree devload diskcopy display dosfsck edit fc fdapm fdisk find format help himemx kernel keyb keyb_lay label localcfg mem mode more move pkg pkgnet shsucdx sort tree');
132
$core_packages_list = explode(' ', 'amb attrib chkdsk choice command cpidos debug deltree devload diskcopy display dosfsck edit fc fdapm fdisk find format help himemx kernel keyb keyb_lay label localcfg mem mode more move pkg pkgnet shsucdx sort tree');
133
 
133
 
134
 
134
 
135
// do a list of all svp packages with their available versions and descriptions
135
// do a list of all svp packages with their available versions and descriptions
136
 
136
 
137
$pkgdb = array();
137
$pkgdb = array();
138
foreach ($pkgfiles as $fname) {
138
foreach ($pkgfiles as $fname) {
139
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
139
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
140
 
140
 
141
  $path_parts = pathinfo($fname);
141
  $path_parts = pathinfo($fname);
142
  $pkgnam = explode('-', $path_parts['filename'])[0];
142
  $pkgnam = explode('-', $path_parts['filename'])[0];
143
  $pkgfullpath = realpath($repodir . '/' . $fname);
143
  $pkgfullpath = realpath($repodir . '/' . $fname);
144
 
144
 
145
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
145
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
146
  if ($lsm == false) {
146
  if ($lsm == false) {
147
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
147
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
148
    continue;
148
    continue;
149
  }
149
  }
150
  $lsmarray = parse_lsm($lsm);
150
  $lsmarray = parse_lsm($lsm);
151
  if (empty($lsmarray['version'])) {
151
  if (empty($lsmarray['version'])) {
152
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
152
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
153
    continue;
153
    continue;
154
  }
154
  }
155
  if (strlen($lsmarray['version']) > 16) {
155
  if (strlen($lsmarray['version']) > 16) {
156
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
156
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
157
    continue;
157
    continue;
158
  }
158
  }
159
  if (empty($lsmarray['description'])) {
159
  if (empty($lsmarray['description'])) {
160
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
160
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
161
    continue;
161
    continue;
162
  }
162
  }
163
 
163
 
164
  // validate the files present in the archive
164
  // validate the files present in the archive
165
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
165
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
166
  $pkgdir = $pkgnam;
166
  $pkgdir = $pkgnam;
167
 
167
 
168
  // special rule for djgpp_* packages: they put their files in djgpp
168
  // special rule for "parent and children" packages
169
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp';
169
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
170
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
170
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
171
 
171
 
-
 
172
  // array used to detect duplicated entries after lower-case conversion
-
 
173
  $duparr = array();
-
 
174
 
172
  foreach ($listoffiles as $f) {
175
  foreach ($listoffiles as $f) {
173
    $f = strtolower($f);
176
    $f = strtolower($f);
-
 
177
    $path_array = explode('/', $f);
-
 
178
    // emit a warning when non-8+3 filenames are spotted and find duplicates
-
 
179
    foreach ($path_array as $item) {
-
 
180
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
-
 
181
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
-
 
182
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
-
 
183
      }
-
 
184
    }
-
 
185
    // look for dups
-
 
186
    if (array_search($f, $duparr) !== false) {
-
 
187
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
-
 
188
    } else {
-
 
189
      $duparr[] = $f;
-
 
190
    }
174
    // LSM file is ok
191
    // LSM file is ok
175
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
192
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
176
    if ($f === "appinfo/") continue;
193
    if ($f === "appinfo/") continue;
177
    // CORE packages are premium citizens and can do a little more
194
    // CORE packages are premium citizens and can do a little more
178
    if (array_search($pkgnam, $core_packages_list) !== false) {
195
    if (array_search($pkgnam, $core_packages_list) !== false) {
179
      if (str_head_is($f, 'bin/')) continue;
196
      if (str_head_is($f, 'bin/')) continue;
180
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
197
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
181
      if ($f === 'doc/') continue;
198
      if ($f === 'doc/') continue;
182
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
199
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
183
      if ($f === 'nls/') continue;
200
      if ($f === 'nls/') continue;
184
    }
201
    }
185
    // well-known "category" dirs are okay
202
    // well-known "category" dirs are okay
186
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
203
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
187
    if ($f === 'progs/') continue;
204
    if ($f === 'progs/') continue;
188
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
205
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
189
    if ($f === 'devel/') continue;
206
    if ($f === 'devel/') continue;
190
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
207
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
191
    if ($f === 'games/') continue;
208
    if ($f === 'games/') continue;
192
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
209
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
193
    if ($f === 'drivers/') continue;
210
    if ($f === 'drivers/') continue;
194
    echo "WARNING: pkg {$fname} contains a file in an illegal location: {$f}\n";
211
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
195
  }
212
  }
196
 
213
 
197
  $meta['fname'] = $fname;
214
  $meta['fname'] = $fname;
198
  $meta['desc'] = $lsmarray['description'];
215
  $meta['desc'] = $lsmarray['description'];
199
 
216
 
200
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
217
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
201
}
218
}
202
 
219
 
203
$db = array();
220
$db = array();
204
 
221
 
205
// iterate over each svp package
222
// iterate over each svp package
206
foreach ($pkgdb as $pkg => $versions) {
223
foreach ($pkgdb as $pkg => $versions) {
207
 
224
 
208
  // sort filenames by version, highest first
225
  // sort filenames by version, highest first
209
  uksort($versions, "version_compare");
226
  uksort($versions, "version_compare");
210
  $versions = array_reverse($versions, true);
227
  $versions = array_reverse($versions, true);
211
 
228
 
212
  foreach ($versions as $ver => $meta) {
229
  foreach ($versions as $ver => $meta) {
213
    $fname = $meta['fname'];
230
    $fname = $meta['fname'];
214
    $desc = $meta['desc'];
231
    $desc = $meta['desc'];
215
 
232
 
216
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
233
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
217
 
234
 
218
    $meta2['ver'] = strval($ver);
235
    $meta2['ver'] = strval($ver);
219
    $meta2['bsum'] = $bsum;
236
    $meta2['bsum'] = $bsum;
220
 
237
 
221
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
238
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
222
    $db[$pkg]['versions'][$fname] = $meta2;
239
    $db[$pkg]['versions'][$fname] = $meta2;
223
  }
240
  }
224
 
241
 
225
  $pkgcount++;
242
  $pkgcount++;
226
 
243
 
227
}
244
}
228
 
245
 
229
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
246
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
230
 
247
 
231
file_put_contents($repodir . '/_index.json', json_encode($db));
248
file_put_contents($repodir . '/_index.json', json_encode($db));
232
 
249
 
233
exit(0);
250
exit(0);
234
 
251
 
235
?>
252
?>
236
 
253