Sync
This commit is contained in:
Ingy döt Net
parent
6f050a029e
commit
776bba907c
3887 changed files with 59894 additions and 7280 deletions
|
|
@ -1,5 +1,4 @@
|
|||
{{Template:clarify task}}
|
||||
|
||||
Often data is produced by one program, in the wrong format for later use by another program or person. In these situations another program can be written to parse and transform the original data into a format useful to the other. The term "Data Munging" is [http://www.google.co.uk/search?q=%22data+munging%22 often] used in programming circles for this task.
|
||||
|
||||
A [http://groups.google.co.uk/group/comp.lang.awk/msg/0ecba3a3fbf247d8?hl=en request] on the comp.lang.awk newsgroup lead to a typical data munging task:
|
||||
|
|
@ -20,7 +19,7 @@ longest period with successive invalid measurements (i.e values with
|
|||
flag<=0)</pre>
|
||||
|
||||
The data is [http://www.eea.europa.eu/help/eea-help-centre/faqs/how-do-i-obtain-eea-reports free to download and use] and is of this format:
|
||||
<pre style="height:20ex;overflow:scroll">
|
||||
<pre style="overflow:scroll">
|
||||
1991-03-30 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1
|
||||
1991-03-31 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 10.000 1 20.000 1 20.000 1 20.000 1 35.000 1 50.000 1 60.000 1 40.000 1 30.000 1 30.000 1 30.000 1 25.000 1 20.000 1 20.000 1 20.000 1 20.000 1 20.000 1 35.000 1
|
||||
1991-03-31 40.000 1 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2 0.000 -2
|
||||
|
|
|
|||
144
Task/Text-processing-1/COBOL/text-processing-1.cobol
Normal file
144
Task/Text-processing-1/COBOL/text-processing-1.cobol
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
IDENTIFICATION DIVISION.
|
||||
PROGRAM-ID. data-munging.
|
||||
|
||||
ENVIRONMENT DIVISION.
|
||||
INPUT-OUTPUT SECTION.
|
||||
FILE-CONTROL.
|
||||
SELECT input-file ASSIGN TO INPUT-FILE-PATH
|
||||
ORGANIZATION LINE SEQUENTIAL
|
||||
FILE STATUS file-status.
|
||||
|
||||
DATA DIVISION.
|
||||
FILE SECTION.
|
||||
FD input-file.
|
||||
01 input-record.
|
||||
03 date-stamp PIC X(10).
|
||||
03 FILLER PIC X.
|
||||
*> Curse whoever decided to use tabs and variable length
|
||||
*> data in the file!
|
||||
03 input-data-pairs PIC X(300).
|
||||
|
||||
WORKING-STORAGE SECTION.
|
||||
78 INPUT-FILE-PATH VALUE "readings.txt".
|
||||
|
||||
01 file-status PIC 99.
|
||||
88 file-is-ok VALUE 0.
|
||||
88 end-of-file VALUE 10.
|
||||
|
||||
01 data-pair.
|
||||
03 val PIC 9(3)V9(3).
|
||||
03 flag PIC S9.
|
||||
88 invalid-flag VALUE -9 THRU 0.
|
||||
|
||||
01 val-length PIC 9.
|
||||
01 flag-length PIC 9.
|
||||
01 offset PIC 99.
|
||||
|
||||
01 day-total PIC 9(5)V9(3).
|
||||
01 grand-total PIC 9(8)V9(3).
|
||||
01 mean-val PIC 9(8)V9(3).
|
||||
|
||||
01 day-rejected PIC 9(5).
|
||||
01 day-accepted PIC 9(5).
|
||||
|
||||
01 total-rejected PIC 9(8).
|
||||
01 total-accepted PIC 9(8).
|
||||
|
||||
01 current-data-gap PIC 9(8).
|
||||
01 max-data-gap PIC 9(8).
|
||||
01 max-data-gap-end PIC X(10).
|
||||
|
||||
PROCEDURE DIVISION.
|
||||
DECLARATIVES.
|
||||
*> Terminate the program if an error occurs on input-file.
|
||||
input-file-error SECTION.
|
||||
USE AFTER STANDARD ERROR ON input-file.
|
||||
|
||||
DISPLAY
|
||||
"An error occurred while reading input.txt. "
|
||||
"File error: " file-status
|
||||
". The program will terminate."
|
||||
END-DISPLAY
|
||||
|
||||
GOBACK
|
||||
.
|
||||
|
||||
END DECLARATIVES.
|
||||
|
||||
main-line.
|
||||
*> Terminate the program if the file cannot be opened.
|
||||
OPEN INPUT input-file
|
||||
IF NOT file-is-ok
|
||||
DISPLAY "File could not be opened. The program will "
|
||||
"terminate."
|
||||
GOBACK
|
||||
END-IF
|
||||
|
||||
*> Process the data in the file.
|
||||
PERFORM FOREVER
|
||||
*> Stop processing if at the end of the file.
|
||||
READ input-file
|
||||
AT END
|
||||
EXIT PERFORM
|
||||
END-READ
|
||||
|
||||
*> Split the data up and process the value-flag pairs.
|
||||
PERFORM UNTIL input-data-pairs = SPACES
|
||||
*> Split off the value-flag pair at the front of the
|
||||
*> record.
|
||||
UNSTRING input-data-pairs DELIMITED BY X"09"
|
||||
INTO val COUNT val-length, flag COUNT flag-length
|
||||
|
||||
COMPUTE offset = val-length + flag-length + 3
|
||||
MOVE input-data-pairs (offset:) TO input-data-pairs
|
||||
|
||||
*> Process according to flag.
|
||||
IF NOT invalid-flag
|
||||
ADD val TO day-total, grand-total
|
||||
|
||||
ADD 1 TO day-accepted, total-accepted
|
||||
|
||||
IF max-data-gap < current-data-gap
|
||||
MOVE current-data-gap TO max-data-gap
|
||||
MOVE date-stamp TO max-data-gap-end
|
||||
END-IF
|
||||
|
||||
MOVE ZERO TO current-data-gap
|
||||
ELSE
|
||||
ADD 1 TO current-data-gap, day-rejected,
|
||||
total-rejected
|
||||
END-IF
|
||||
END-PERFORM
|
||||
|
||||
*> Display day stats.
|
||||
DIVIDE day-total BY day-accepted GIVING mean-val
|
||||
DISPLAY
|
||||
date-stamp
|
||||
" Reject: " day-rejected
|
||||
" Accept: " day-accepted
|
||||
" Average: " mean-val
|
||||
END-DISPLAY
|
||||
|
||||
INITIALIZE day-rejected, day-accepted, mean-val,
|
||||
day-total
|
||||
END-PERFORM
|
||||
|
||||
CLOSE input-file
|
||||
|
||||
*> Display overall stats.
|
||||
DISPLAY SPACE
|
||||
DISPLAY "File: " INPUT-FILE-PATH
|
||||
DISPLAY "Total: " grand-total
|
||||
DISPLAY "Readings: " total-accepted
|
||||
|
||||
DIVIDE grand-total BY total-accepted GIVING mean-val
|
||||
DISPLAY "Average: " mean-val
|
||||
|
||||
DISPLAY SPACE
|
||||
DISPLAY "Bad readings: " total-rejected
|
||||
DISPLAY "Maximum number of consecutive bad readings is "
|
||||
max-data-gap
|
||||
DISPLAY "Ends on date " max-data-gap-end
|
||||
|
||||
GOBACK
|
||||
.
|
||||
|
|
@ -1,101 +1,48 @@
|
|||
(defstruct (measurement
|
||||
(:conc-name "MEASUREMENT-")
|
||||
(:constructor make-measurement (counter line date flag value)))
|
||||
(counter 0 :type (integer 0))
|
||||
(line 0 :type (integer 0))
|
||||
(date nil :type symbol)
|
||||
(flag 0 :type integer)
|
||||
(value 0 :type real))
|
||||
(defvar *invalid-count*)
|
||||
(defvar *max-invalid*)
|
||||
(defvar *max-invalid-date*)
|
||||
(defvar *total-sum*)
|
||||
(defvar *total-valid*)
|
||||
|
||||
(defun measurement-valid-p (m)
|
||||
(> (measurement-flag m) 0))
|
||||
(defun read-flag (stream date)
|
||||
(let ((flag (read stream)))
|
||||
(if (plusp flag)
|
||||
(setf *invalid-count* 0)
|
||||
(when (< *max-invalid* (incf *invalid-count*))
|
||||
(setf *max-invalid* *invalid-count*)
|
||||
(setf *max-invalid-date* date)))
|
||||
flag))
|
||||
|
||||
(defun map-data-stream (function stream)
|
||||
(flet ((scan (&optional (errorp t)) (read stream errorp nil)))
|
||||
(loop
|
||||
:with global-count = 0
|
||||
:for date = (scan nil) :then (scan nil)
|
||||
:for line-number :upfrom 1
|
||||
:while date
|
||||
:do (loop
|
||||
:for count :upfrom 0 :below 24
|
||||
:do (let* ((value (scan)) (flag (scan)))
|
||||
(funcall function (make-measurement global-count line-number date flag value))
|
||||
(incf global-count)))
|
||||
:finally (return global-count))))
|
||||
(defun parse-line (line)
|
||||
(with-input-from-string (s line)
|
||||
(let ((date (make-string 10)))
|
||||
(read-sequence date s)
|
||||
(cons date (loop repeat 24 collect (list (read s)
|
||||
(read-flag s date)))))))
|
||||
|
||||
(defun map-data-file (function pathname)
|
||||
(with-open-file (stream pathname
|
||||
:element-type 'character
|
||||
:direction :input
|
||||
:if-does-not-exist :error)
|
||||
(map-data-stream function stream)))
|
||||
(defun analyze-line (line)
|
||||
(destructuring-bind (date &rest rest) line
|
||||
(let* ((valid (remove-if-not #'plusp rest :key #'second))
|
||||
(n (length valid))
|
||||
(sum (apply #'+ (mapcar #'rationalize (mapcar #'first valid))))
|
||||
(avg (if valid (/ sum n) 0)))
|
||||
(incf *total-valid* n)
|
||||
(incf *total-sum* sum)
|
||||
(format t "Line: ~a Reject: ~2d Accept: ~2d ~
|
||||
Line_tot: ~8,3f Line_avg: ~7,3f~%"
|
||||
date (- 24 n) n sum avg))))
|
||||
|
||||
(defmacro do-data-stream ((variable stream) &body body)
|
||||
`(map-data-stream
|
||||
(lambda (,variable) ,@body)
|
||||
,stream))
|
||||
|
||||
(defmacro do-data-file ((variable file) &body body)
|
||||
`(map-data-file
|
||||
(lambda (,variable) ,@body)
|
||||
,file))
|
||||
|
||||
(let ((current-day nil)
|
||||
(current-line 0)
|
||||
(beginning-of-misreadings nil)
|
||||
(current-length 0)
|
||||
(worst-beginning nil)
|
||||
(worst-length 0)
|
||||
(sum-of-day 0)
|
||||
(count-of-day 0))
|
||||
|
||||
(flet ((write-end-of-day-report ()
|
||||
(when current-day
|
||||
(format t "Line ~5D Date ~A: Accepted ~2D Total ~8,3F Average ~8,3F~%"
|
||||
current-line current-day count-of-day sum-of-day
|
||||
(if (> count-of-day 0) (/ sum-of-day count-of-day) sum-of-day)))))
|
||||
|
||||
(do-data-file (m #P"D:/Scratch/data.txt")
|
||||
|
||||
(let* ((date (measurement-date m))
|
||||
(line-number (measurement-line m))
|
||||
(validp (measurement-valid-p m))
|
||||
(day-changed-p (/= current-line line-number))
|
||||
(value (measurement-value m)))
|
||||
|
||||
(when day-changed-p
|
||||
(write-end-of-day-report)
|
||||
(setf current-day date)
|
||||
(setf current-line line-number)
|
||||
(setf sum-of-day 0)
|
||||
(setf count-of-day 0))
|
||||
|
||||
(if (not validp)
|
||||
(if beginning-of-misreadings
|
||||
(incf current-length)
|
||||
(progn
|
||||
(setf beginning-of-misreadings m)
|
||||
(setf current-length 1)))
|
||||
(progn
|
||||
(when beginning-of-misreadings
|
||||
(if (> current-length worst-length)
|
||||
(progn
|
||||
(setf worst-beginning beginning-of-misreadings)
|
||||
(setf worst-length current-length))
|
||||
(progn
|
||||
(setf beginning-of-misreadings nil)
|
||||
(setf current-length 0))))
|
||||
(incf sum-of-day value)
|
||||
(incf count-of-day)))))
|
||||
|
||||
(when (and beginning-of-misreadings (> current-length worst-length))
|
||||
(setf worst-beginning beginning-of-misreadings)
|
||||
(setf worst-length current-length))
|
||||
|
||||
(write-end-of-day-report))
|
||||
|
||||
(format t "Worst run started ~A (~D) and has length ~D~%"
|
||||
(measurement-date worst-beginning)
|
||||
(measurement-counter worst-beginning)
|
||||
worst-length))
|
||||
(defun process (pathname)
|
||||
(let ((*invalid-count* 0) (*max-invalid* 0) *max-invalid-date*
|
||||
(*total-sum* 0) (*total-valid* 0))
|
||||
(with-open-file (f pathname)
|
||||
(loop for line = (read-line f nil nil)
|
||||
while line
|
||||
do (analyze-line (parse-line line))))
|
||||
(format t "~%File = ~a" pathname)
|
||||
(format t "~&Total = ~f" *total-sum*)
|
||||
(format t "~&Readings = ~a" *total-valid*)
|
||||
(format t "~&Average = ~10,3f~%" (/ *total-sum* *total-valid*))
|
||||
(format t "~%Maximum run(s) of ~a consecutive false readings ends at ~
|
||||
line starting with date(s): ~a~%"
|
||||
*max-invalid* *max-invalid-date*)))
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import std.stdio, std.conv, std.string;
|
||||
void main(in string[] args) {
|
||||
import std.stdio, std.conv, std.string;
|
||||
|
||||
void main(string[] args) {
|
||||
/*const*/ auto fileNames = (args.length == 1) ? ["readings.txt"] :
|
||||
args[1 .. $];
|
||||
const fileNames = (args.length == 1) ? ["readings.txt"] :
|
||||
args[1 .. $];
|
||||
|
||||
int noData, noDataMax = -1;
|
||||
string[] noDataMaxLine;
|
||||
|
|
@ -10,13 +10,13 @@ void main(string[] args) {
|
|||
double fileTotal = 0.0;
|
||||
int fileValues;
|
||||
|
||||
foreach (fileName; fileNames) {
|
||||
foreach (char[] line; File(fileName).byLine()) {
|
||||
foreach (const fileName; fileNames) {
|
||||
foreach (char[] line; fileName.File.byLine) {
|
||||
double lineTotal = 0.0;
|
||||
int lineValues;
|
||||
|
||||
// Extract field info
|
||||
const parts = line.split();
|
||||
// Extract field info.
|
||||
const parts = line.split;
|
||||
const date = parts[0];
|
||||
const fields = parts[1 .. $];
|
||||
assert(fields.length % 2 == 0,
|
||||
|
|
@ -24,15 +24,15 @@ void main(string[] args) {
|
|||
fields.length));
|
||||
|
||||
for (int i; i < fields.length; i += 2) {
|
||||
immutable value = to!double(fields[i]);
|
||||
immutable flag = to!int(fields[i + 1]);
|
||||
immutable value = fields[i].to!double;
|
||||
immutable flag = fields[i + 1].to!int;
|
||||
|
||||
if (flag < 1) {
|
||||
noData++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check run of data-absent fields
|
||||
// Check run of data-absent fields.
|
||||
if (noDataMax == noData && noData > 0)
|
||||
noDataMaxLine ~= date.idup;
|
||||
|
||||
|
|
@ -42,15 +42,15 @@ void main(string[] args) {
|
|||
noDataMaxLine[0] = date.idup;
|
||||
}
|
||||
|
||||
// Re-initialise run of noData counter
|
||||
// Re-initialise run of noData counter.
|
||||
noData = 0;
|
||||
|
||||
// Gather values for averaging
|
||||
// Gather values for averaging.
|
||||
lineTotal += value;
|
||||
lineValues++;
|
||||
}
|
||||
|
||||
// Totals for the file so far
|
||||
// Totals for the file so far.
|
||||
fileTotal += lineTotal;
|
||||
fileValues += lineValues;
|
||||
|
||||
|
|
@ -64,12 +64,12 @@ void main(string[] args) {
|
|||
}
|
||||
}
|
||||
|
||||
writeln("\nFile(s) = ", fileNames.join(", "));
|
||||
writefln("\nFile(s) = %-(%s, %)", fileNames);
|
||||
writefln("Total = %10.3f", fileTotal);
|
||||
writefln("Readings = %6d", fileValues);
|
||||
writefln("Average = %10.3f", fileTotal / fileValues);
|
||||
|
||||
writefln("\nMaximum run(s) of %d consecutive false " ~
|
||||
"readings ends at line starting with date(s): %s",
|
||||
noDataMax, join(noDataMaxLine, ", "));
|
||||
"readings ends at line starting with date(s): %-(%s, %)",
|
||||
noDataMax, noDataMaxLine);
|
||||
}
|
||||
|
|
|
|||
66
Task/Text-processing-1/Erlang/text-processing-1.erl
Normal file
66
Task/Text-processing-1/Erlang/text-processing-1.erl
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
-module( text_processing ).
|
||||
|
||||
-export( [file_contents/1, main/1] ).
|
||||
|
||||
-record( acc, {failed={"", 0, 0}, files=[], ok=0, total=0} ).
|
||||
|
||||
file_contents( Name ) ->
|
||||
{ok, Binary} = file:read_file( Name ),
|
||||
[line_contents(X) || X <- binary:split(Binary, <<"\r\n">>, [global]), X =/= <<>>].
|
||||
|
||||
main( Files ) ->
|
||||
Acc = lists:foldl( fun file/2, #acc{}, Files ),
|
||||
{Failed_date, Failed, _Continuation} = Acc#acc.failed,
|
||||
io:fwrite( "~nFile(s)=~p~nTotal=~.2f~nReadings=~p~nAverage=~.2f~n~nMaximum run(s) of ~p consecutive false readings ends at line starting with date(s): ~p~n",
|
||||
[lists:reverse(Acc#acc.files), Acc#acc.total, Acc#acc.ok, Acc#acc.total / Acc#acc.ok, Failed, Failed_date] ).
|
||||
|
||||
|
||||
|
||||
file( Name, #acc{files=Files}=Acc ) ->
|
||||
try
|
||||
Line_contents = file_contents( Name ),
|
||||
lists:foldl( fun file_content_line/2, Acc#acc{files=[Name | Files]}, Line_contents )
|
||||
|
||||
catch
|
||||
_:Error ->
|
||||
io:fwrite( "Error: Failed to read ~s: ~p~n", [Name, Error] ),
|
||||
Acc
|
||||
end.
|
||||
|
||||
file_content_line( {Date, Value_flags}, #acc{failed=Failed, ok=Ok, total=Total}=Acc ) ->
|
||||
New_failed = file_content_line_failed( Value_flags, Date, Failed ),
|
||||
{Sum, Oks, Average} = file_content_line_oks_0( [X || {X, ok} <- Value_flags] ),
|
||||
io:fwrite( "Line=~p\tRejected=~p\tAccepted=~p\tLine total=~.2f\tLine average=~.2f~n", [Date, erlang:length(Value_flags) - Oks, Oks, Sum, Average] ),
|
||||
Acc#acc{failed=New_failed, ok=Ok + Oks, total=Total + Sum}.
|
||||
|
||||
file_content_line_failed( [], Date, {_Failed_date, Failed, Acc} ) when Acc > Failed ->
|
||||
{Date, Acc, Acc};
|
||||
file_content_line_failed( [], _Date, Failed ) ->
|
||||
Failed;
|
||||
file_content_line_failed( [{_V, error} | T], Date, {Failed_date, Failed, Acc} ) ->
|
||||
file_content_line_failed( T, Date, {Failed_date, Failed, Acc + 1} );
|
||||
file_content_line_failed( [_H | T], Date, {_Failed_date, Failed, Acc} ) when Acc > Failed ->
|
||||
file_content_line_failed( T, Date, {Date, Acc, 0} );
|
||||
file_content_line_failed( [_H | T], Date, {Failed_date, Failed, _Acc} ) ->
|
||||
file_content_line_failed( T, Date, {Failed_date, Failed, 0} ).
|
||||
|
||||
file_content_line_flag( N ) when N > 0 -> ok;
|
||||
file_content_line_flag( _N ) -> error.
|
||||
|
||||
file_content_line_oks_0( [] ) -> {0.0, 0, 0.0};
|
||||
file_content_line_oks_0( Ok_value_flags ) ->
|
||||
Sum = lists:sum( Ok_value_flags ),
|
||||
Oks = erlang:length( Ok_value_flags ),
|
||||
{Sum, Oks, Sum / Oks}.
|
||||
|
||||
file_content_line_value_flag( Binary, {[], Acc} ) ->
|
||||
Flag = file_content_line_flag( erlang:list_to_integer(binary:bin_to_list(Binary)) ),
|
||||
{[Flag], Acc};
|
||||
file_content_line_value_flag( Binary, {[Flag], Acc} ) ->
|
||||
Value = erlang:list_to_float( binary:bin_to_list(Binary) ),
|
||||
{[], [{Value, Flag} | Acc]}.
|
||||
|
||||
line_contents( Line ) ->
|
||||
[Date_binary | Rest] = binary:split( Line, <<"\t">>, [global] ),
|
||||
{_Previous, Value_flags} = lists:foldr( fun file_content_line_value_flag/2, {[], []}, Rest ), % Preserve order
|
||||
{binary:bin_to_list( Date_binary ), Value_flags}.
|
||||
Loading…
Add table
Add a link
Reference in a new issue