This commit is contained in:
Ingy döt Net 2013-10-27 22:24:23 +00:00
parent 6f050a029e
commit 776bba907c
3887 changed files with 59894 additions and 7280 deletions

View file

@ -0,0 +1,64 @@
void
check_format(list l)
{
integer i;
text s;
if (l_length(l) != 49) {
error("wrong number of fields");
}
s = lf_q_text(l);
if (length(s) != 10 || character(s, 4) != '-' || character(s, 7) != '-') {
error("bad date format");
}
atoi(delete(delete(s, 7), 4));
i = 1;
while (i < 49) {
l_r_real(l, i, atof(l_q_text(l, i)));
i += 1;
l_r_integer(l, i, atoi(l_q_text(l, i)));
i += 1;
}
}
integer
main(void)
{
integer goods;
file f;
list l;
record r;
goods = 0;
f_affix(f, "readings.txt");
while (f_list(f, l, 0) != -1) {
if (!trap(check_format, l)) {
if (r_key(r, l_head(l))) {
v_text(cat3("duplicate ", l_head(l), " line\n"));
} else {
integer i;
r_put(r, l_head(l), 0);
i = 2;
while (i < 49) {
if (l_q_integer(l, i) != 1) {
break;
}
i += 2;
}
if (49 < i) {
goods += 1;
}
}
}
}
o_integer(goods);
o_text(" good unique lines\n");
return 0;
}

View file

@ -0,0 +1,84 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
typedef struct { const char *s; int ln, bad; } rec_t;
int cmp_rec(const void *aa, const void *bb)
{
const rec_t *a = aa, *b = bb;
return a->s == b->s ? 0 : !a->s ? 1 : !b->s ? -1 : strncmp(a->s, b->s, 10);
}
int read_file(const char *fn)
{
int fd = open(fn, O_RDONLY);
if (fd == -1) return 0;
struct stat s;
fstat(fd, &s);
char *txt = malloc(s.st_size);
read(fd, txt, s.st_size);
close(fd);
int i, j, lines = 0, k, di, bad;
for (i = lines = 0; i < s.st_size; i++)
if (txt[i] == '\n') {
txt[i] = '\0';
lines++;
}
rec_t *rec = calloc(sizeof(rec_t), lines);
const char *ptr, *end;
rec[0].s = txt;
rec[0].ln = 1;
for (i = 0; i < lines; i++) {
if (i + 1 < lines) {
rec[i + 1].s = rec[i].s + strlen(rec[i].s) + 1;
rec[i + 1].ln = i + 2;
}
if (sscanf(rec[i].s, "%4d-%2d-%2d", &di, &di, &di) != 3) {
printf("bad line %d: %s\n", i, rec[i].s);
rec[i].s = 0;
continue;
}
ptr = rec[i].s + 10;
for (j = k = 0; j < 25; j++) {
if (!strtod(ptr, (char**)&end) && end == ptr) break;
k++, ptr = end;
if (!(di = strtol(ptr, (char**)&end, 10)) && end == ptr) break;
k++, ptr = end;
if (di < 1) rec[i].bad = 1;
}
if (k != 48) {
printf("bad format at line %d: %s\n", i, rec[i].s);
rec[i].s = 0;
}
}
qsort(rec, lines, sizeof(rec_t), cmp_rec);
for (i = 1, bad = rec[0].bad, j = 0; i < lines && rec[i].s; i++) {
if (rec[i].bad) bad++;
if (strncmp(rec[i].s, rec[j].s, 10)) {
j = i;
} else
printf("dup line %d: %.10s\n", rec[i].ln, rec[i].s);
}
free(rec);
free(txt);
printf("\n%d out %d lines good\n", lines - bad, lines);
return 0;
}
int main()
{
read_file("readings.txt");
return 0;
}

View file

@ -0,0 +1,163 @@
IDENTIFICATION DIVISION.
PROGRAM-ID. text-processing-2.
ENVIRONMENT DIVISION.
INPUT-OUTPUT SECTION.
FILE-CONTROL.
SELECT readings ASSIGN Input-File-Path
ORGANIZATION LINE SEQUENTIAL
FILE STATUS file-status.
DATA DIVISION.
FILE SECTION.
FD readings.
01 reading-record.
03 date-stamp PIC X(10).
03 FILLER PIC X.
03 input-data PIC X(300).
LOCAL-STORAGE SECTION.
78 Input-File-Path VALUE "readings.txt".
78 Num-Data-Points VALUE 48.
01 file-status PIC XX.
01 current-line PIC 9(5).
01 num-date-stamps-read PIC 9(5).
01 read-date-stamps-area.
03 read-date-stamps PIC X(10) OCCURS 1 TO 10000 TIMES
DEPENDING ON num-date-stamps-read
INDEXED BY date-stamp-idx.
01 offset PIC 999.
01 data-len PIC 999.
01 data-flag PIC X.
88 data-not-found VALUE "N".
01 data-field PIC X(25).
01 i PIC 99.
01 num-good-readings PIC 9(5).
01 reading-flag PIC X.
88 bad-reading VALUE "B".
01 delim PIC X.
PROCEDURE DIVISION.
DECLARATIVES.
readings-error SECTION.
USE AFTER ERROR ON readings
DISPLAY "An error occurred while using " Input-File-Path
DISPLAY "Error code " file-status
DISPLAY "The program will terminate."
CLOSE readings
GOBACK
.
END DECLARATIVES.
main-line.
OPEN INPUT readings
*> Process each line of the file.
PERFORM FOREVER
READ readings
AT END
EXIT PERFORM
END-READ
ADD 1 TO current-line
IF reading-record = SPACES
DISPLAY "Line " current-line " is blank."
EXIT PERFORM CYCLE
END-IF
PERFORM check-duplicate-date-stamp
*> Check there are 24 data pairs and see if all the
*> readings are ok.
INITIALIZE offset, reading-flag, data-flag
PERFORM VARYING i FROM 1 BY 1 UNTIL Num-Data-Points < i
PERFORM get-next-field
IF data-not-found
DISPLAY "Line " current-line " has missing "
"fields."
SET bad-reading TO TRUE
EXIT PERFORM
END-IF
*> Every other data field is the instrument flag.
IF FUNCTION MOD(i, 2) = 0 AND NOT bad-reading
IF FUNCTION NUMVAL(data-field) <= 0
SET bad-reading TO TRUE
END-IF
END-IF
ADD data-len TO offset
END-PERFORM
IF NOT bad-reading
ADD 1 TO num-good-readings
END-IF
END-PERFORM
CLOSE readings
*> Display results.
DISPLAY SPACE
DISPLAY current-line " lines read."
DISPLAY num-good-readings " have good readings for all "
"instruments."
GOBACK
.
check-duplicate-date-stamp.
SEARCH read-date-stamps
AT END
ADD 1 TO num-date-stamps-read
MOVE date-stamp
TO read-date-stamps (num-date-stamps-read)
WHEN read-date-stamps (date-stamp-idx) = date-stamp
DISPLAY "Date " date-stamp " is duplicated at "
"line " current-line "."
END-SEARCH
.
get-next-field.
INSPECT input-data (offset:) TALLYING offset
FOR LEADING X"09"
*> The fields are normally delimited by a tab.
MOVE X"09" TO delim
PERFORM find-num-chars-before-delim
*> If the delimiter was not found...
IF FUNCTION SUM(data-len, offset) > 300
*> The data may be delimited by a space if it is at the
*> end of the line.
MOVE SPACE TO delim
PERFORM find-num-chars-before-delim
IF FUNCTION SUM(data-len, offset) > 300
SET data-not-found TO TRUE
EXIT PARAGRAPH
END-IF
END-IF
IF data-len = 0
SET data-not-found TO TRUE
EXIT PARAGRAPH
END-IF
MOVE input-data (offset:data-len) TO data-field
.
find-num-chars-before-delim.
INITIALIZE data-len
INSPECT input-data (offset:) TALLYING data-len
FOR CHARACTERS BEFORE delim
.

View file

@ -1,27 +1,27 @@
import std.stdio, std.array, std.string, std.regex, std.conv,
std.algorithm;
void main() {
// works but eats lot of RAM in DMD 2.059
//const rxDate = ctRegex!(`^\d\d\d\d-\d\d-\d\d$`);
auto rxDate = regex(`^\d\d\d\d-\d\d-\d\d$`);
import std.stdio, std.array, std.string, std.regex, std.conv,
std.algorithm;
auto rxDate = `^\d\d\d\d-\d\d-\d\d$`.regex;
// Works but eats lot of RAM in DMD 2.064.
// auto rxDate = ctRegex!(`^\d\d\d\d-\d\d-\d\d$`);
int[string] repeatedDates;
int goodReadings;
foreach (string line; lines(File("readings.txt"))) {
foreach (string line; "readings.txt".File.lines) {
try {
auto parts = line.split();
auto parts = line.split;
if (parts.length != 49)
throw new Exception("Wrong column count");
if (match(parts[0], rxDate).empty)
if (parts[0].match(rxDate).empty)
throw new Exception("Date is wrong");
repeatedDates[parts[0]]++;
bool noProblem = true;
for (int i = 1; i < 48; i += 2) {
if (to!int(parts[i + 1]) < 1)
if (parts[i + 1].to!int < 1)
// don't break loop because it's validation too.
noProblem = false;
if (!isNumeric(parts[i]))
if (!parts[i].isNumeric)
throw new Exception("Reading is wrong: "~parts[i]);
}
if (noProblem)
@ -31,8 +31,7 @@ void main() {
}
}
writeln("Duplicated timestamps: ",
repeatedDates.keys.filter!(k => repeatedDates[k] > 1)().
join(", "));
writefln("Duplicated timestamps: %-(%s, %)",
repeatedDates.byKey.filter!(k => repeatedDates[k] > 1));
writeln("Good reading records: ", goodReadings);
}

View file

@ -0,0 +1,31 @@
-module( text_processing2 ).
-export( [task/0] ).
task() ->
Name = "priv/readings.txt",
try
File_contents = text_processing:file_contents( Name ),
[correct_field_format(X) || X<- File_contents],
{_Previous, Duplicates} = lists:foldl( fun date_duplicates/2, {"", []}, File_contents ),
io:fwrite( "Duplicates: ~p~n", [Duplicates] ),
Good = [X || X <- File_contents, is_all_good_readings(X)],
io:fwrite( "Good readings: ~p~n", [erlang:length(Good)] )
catch
_:Error ->
io:fwrite( "Error: Failed when checking ~s: ~p~n", [Name, Error] )
end.
correct_field_format( {_Date, Value_flags} ) ->
Corret_number = value_flag_records(),
{correct_field_format, Corret_number} = {correct_field_format, erlang:length(Value_flags)}.
date_duplicates( {Date, _Value_flags}, {Date, Acc} ) -> {Date, [Date | Acc]};
date_duplicates( {Date, _Value_flags}, {_Other, Acc} ) -> {Date, Acc}.
is_all_good_readings( {_Date, Value_flags} ) -> value_flag_records() =:= erlang:length( [ok || {_Value, ok} <- Value_flags] ).
value_flag_records() -> 24.

View file

@ -0,0 +1,33 @@
procedure main(A)
dups := set()
goodRecords := 0
lastDate := badFile := &null
f := A[1] | "readings.txt"
fin := open(f) | stop("Cannot open file '",f,"'")
while (fields := 0, badReading := &null, line := read(fin)) do {
line ? {
ldate := tab(many(&digits ++ '-')) | (badFile := "yes", next)
if \lastDate == ldate then insert(dups, ldate)
lastDate := ldate
while tab(many(' \t')) do {
(value := real(tab(many(&digits++'-.'))),
tab(many(' \t')),
flag := integer(tab(many(&digits++'-'))),
fields +:= 1) | (badFile := "yes")
if flag < 1 then badReading := "yes"
}
}
if fields = 24 then goodRecords +:= (/badReading, 1)
else badFile := "yes"
}
if (\badFile) then write(f," has field format issues.")
write("There are ",goodRecords," records with all good readings.")
if *dups > 0 then {
write("The following dates have multiple records:")
every writes(" ",!sort(dups))
write()
}
end