2016 Update

This commit is contained in:
Tina Müller 2016-12-05 22:15:40 +01:00
parent 948b86eafa
commit dcf5d15da3
7965 changed files with 139854 additions and 31002 deletions

View file

@ -1,10 +1,14 @@
[[wp:Comma-separated values|CSV spreadsheet files]] are suitable for
storing tabular data in a relatively portable way. The CSV format is
flexible but somewhat ill-defined. For present purposes, authors may
assume that the data fields contain no commas, backslashes, or
quotation marks.
[[wp:Comma-separated values|CSV spreadsheet files]] are suitable for storing tabular data in a relatively portable way.
The task here is to read a CSV file, change some values and save the changes back to a file. For this task we will use the following CSV file:
The CSV format is flexible but somewhat ill-defined.
For present purposes, authors may assume that the data fields contain no commas, backslashes, or quotation marks.
;Task:
Read a CSV file, change some values and save the changes back to a file.
For this task we will use the following CSV file:
C1,C2,C3,C4,C5
1,5,9,13,17
@ -17,3 +21,4 @@ The task here is to read a CSV file, change some values and save the changes bac
<li/> Show how to add a column, headed 'SUM', of the sums of the rows.
<li/> If possible, illustrate the use of built-in or standard functions, methods, or libraries, that handle generic CSV files.
</ul>
<br><br>

View file

@ -0,0 +1,275 @@
#define TITLE "CSV data manipulation"
#define URL "http://rosettacode.org/wiki/CSV_data_manipulation"
#define _GNU_SOURCE
#define bool int
#include <stdio.h>
#include <stdlib.h> /* malloc...*/
#include <string.h> /* strtok...*/
#include <ctype.h>
#include <errno.h>
/**
* How to read a CSV file ?
*/
typedef struct {
char * delim;
unsigned int rows;
unsigned int cols;
char ** table;
} CSV;
/**
* Utility function to trim whitespaces from left & right of a string
*/
int trim(char ** str) {
int trimmed;
int n;
int len;
len = strlen(*str);
n = len - 1;
/* from right */
while((n>=0) && isspace((*str)[n])) {
(*str)[n] = '\0';
trimmed += 1;
n--;
}
/* from left */
n = 0;
while((n < len) && (isspace((*str)[0]))) {
(*str)[0] = '\0';
*str = (*str)+1;
trimmed += 1;
n++;
}
return trimmed;
}
/**
* De-allocate csv structure
*/
int csv_destroy(CSV * csv) {
if (csv == NULL) { return 0; }
if (csv->table != NULL) { free(csv->table); }
if (csv->delim != NULL) { free(csv->delim); }
free(csv);
return 0;
}
/**
* Allocate memory for a CSV structure
*/
CSV * csv_create(unsigned int cols, unsigned int rows) {
CSV * csv;
csv = malloc(sizeof(CSV));
csv->rows = rows;
csv->cols = cols;
csv->delim = strdup(",");
csv->table = malloc(sizeof(char *) * cols * rows);
if (csv->table == NULL) { goto error; }
memset(csv->table, 0, sizeof(char *) * cols * rows);
return csv;
error:
csv_destroy(csv);
return NULL;
}
/**
* Get value in CSV table at COL, ROW
*/
char * csv_get(CSV * csv, unsigned int col, unsigned int row) {
unsigned int idx;
idx = col + (row * csv->cols);
return csv->table[idx];
}
/**
* Set value in CSV table at COL, ROW
*/
int csv_set(CSV * csv, unsigned int col, unsigned int row, char * value) {
unsigned int idx;
idx = col + (row * csv->cols);
csv->table[idx] = value;
return 0;
}
void csv_display(CSV * csv) {
int row, col;
char * content;
if ((csv->rows == 0) || (csv->cols==0)) {
printf("[Empty table]\n");
return ;
}
printf("\n[Table cols=%d rows=%d]\n", csv->cols, csv->rows);
for (row=0; row<csv->rows; row++) {
printf("[|");
for (col=0; col<csv->cols; col++) {
content = csv_get(csv, col, row);
printf("%s\t|", content);
}
printf("]\n");
}
printf("\n");
}
/**
* Resize CSV table
*/
int csv_resize(CSV * old_csv, unsigned int new_cols, unsigned int new_rows) {
unsigned int cur_col,
cur_row,
max_cols,
max_rows;
CSV * new_csv;
char * content;
bool in_old, in_new;
/* Build a new (fake) csv */
new_csv = csv_create(new_cols, new_rows);
if (new_csv == NULL) { goto error; }
new_csv->rows = new_rows;
new_csv->cols = new_cols;
max_cols = (new_cols > old_csv->cols)? new_cols : old_csv->cols;
max_rows = (new_rows > old_csv->rows)? new_rows : old_csv->rows;
for (cur_col=0; cur_col<max_cols; cur_col++) {
for (cur_row=0; cur_row<max_rows; cur_row++) {
in_old = (cur_col < old_csv->cols) && (cur_row < old_csv->rows);
in_new = (cur_col < new_csv->cols) && (cur_row < new_csv->rows);
if (in_old && in_new) {
/* re-link data */
content = csv_get(old_csv, cur_col, cur_row);
csv_set(new_csv, cur_col, cur_row, content);
} else if (in_old) {
/* destroy data */
content = csv_get(old_csv, cur_col, cur_row);
free(content);
} else { /* skip */ }
}
}
/* on rows */
free(old_csv->table);
old_csv->rows = new_rows;
old_csv->cols = new_cols;
old_csv->table = new_csv->table;
new_csv->table = NULL;
csv_destroy(new_csv);
return 0;
error:
printf("Unable to resize CSV table: error %d - %s\n", errno, strerror(errno));
return -1;
}
/**
* Open CSV file and load its content into provided CSV structure
**/
int csv_open(CSV * csv, char * filename) {
FILE * fp;
unsigned int m_rows;
unsigned int m_cols, cols;
char line[2048];
char * lineptr;
char * token;
fp = fopen(filename, "r");
if (fp == NULL) { goto error; }
m_rows = 0;
m_cols = 0;
while(fgets(line, sizeof(line), fp) != NULL) {
m_rows += 1;
cols = 0;
lineptr = line;
while ((token = strtok(lineptr, csv->delim)) != NULL) {
lineptr = NULL;
trim(&token);
cols += 1;
if (cols > m_cols) { m_cols = cols; }
csv_resize(csv, m_cols, m_rows);
csv_set(csv, cols-1, m_rows-1, strdup(token));
}
}
fclose(fp);
csv->rows = m_rows;
csv->cols = m_cols;
return 0;
error:
fclose(fp);
printf("Unable to open %s for reading.", filename);
return -1;
}
/**
* Open CSV file and save CSV structure content into it
**/
int csv_save(CSV * csv, char * filename) {
FILE * fp;
int row, col;
char * content;
fp = fopen(filename, "w");
for (row=0; row<csv->rows; row++) {
for (col=0; col<csv->cols; col++) {
content = csv_get(csv, col, row);
fprintf(fp, "%s%s", content,
((col == csv->cols-1) ? "" : csv->delim) );
}
fprintf(fp, "\n");
}
fclose(fp);
return 0;
}
/**
* Test
*/
int main(int argc, char ** argv) {
CSV * csv;
printf("%s\n%s\n\n",TITLE, URL);
csv = csv_create(0, 0);
csv_open(csv, "fixtures/csv-data-manipulation.csv");
csv_display(csv);
csv_set(csv, 0, 0, "Column0");
csv_set(csv, 1, 1, "100");
csv_set(csv, 2, 2, "200");
csv_set(csv, 3, 3, "300");
csv_set(csv, 4, 4, "400");
csv_display(csv);
csv_save(csv, "tmp/csv-data-manipulation.result.csv");
csv_destroy(csv);
return 0;
}

View file

@ -1,35 +1,39 @@
(defun csv-to-nested-list (filename seperator)
"Reads the csv to a nested lisp list, where each sublist represents a line.
Each line is read in as a string, the commas are substituted by spaces and
parantheses are added to the beginning and the end. Then the string can be interpreted by the
reader as an actual lisp list. A nested lisp containing all sub-lists (lines) is returned.
First line is assumed to be a comment (as no comment syntax is specified)."
(let ((list nil))
(with-open-file (input filename)
(setf list
(loop for line = (read-line input nil)
while line collect (read-from-string
(substitute #\Tab #\, (format nil "(~a)~%" line)))))
;; throw away first line, which is assumed to be a comment
(cdr list))))
(defun csvfile-to-nested-list (filename delim-char)
"Reads the csv to a nested list, where each sublist represents a line."
(with-open-file (input filename)
(loop :for line := (read-line input nil) :while line
:collect (read-from-string
(substitute #\SPACE delim-char
(format nil "(~a)~%" line))))))
(defun calc-sums (nested-list)
"Return a list of sums of each sub-list in a nested list."
(loop for sublist in nested-list collect (apply #'+ sublist)))
(defun sublist-sum-list (nested-list)
"Return a list with the sum of each list of numbers in a nested list."
(mapcar (lambda (l) (if (every #'numberp l)
(reduce #'+ l) nil))
nested-list))
(defun list-to-csv (nested-list)
(defun append-each-sublist (nested-list1 nested-list2)
"Horizontally append the sublists in two nested lists. Used to add columns."
(mapcar #'append nested-list1 nested-list2))
(defun nested-list-to-csv (nested-list delim-string)
"Converts the nested list back into a csv-formatted string."
(substitute #\, #\
(substitute #\newline #\)
(remove #\((string-trim ")(" (format nil "~A" nested-list))))))
;; main program
;; prints the results as lisp lists and as csv
(let ((nested-list (csv-to-nested-list "example_comma_csv.txt" #\,))
(sum-list nil)
(comment "#C1,C2,C3,C4,C5,SUM"))
(setf sum-list (loop
for list in nested-list
for sum in (calc-sums nested-list)
collect (append list (list sum))))
(format t "~A~%~%" sum-list) ;; print nested list in lisp representation
(format t "~A~%~A~%" comment (list-to-csv sum-list))) ;; print it again as csv
(format nil (concatenate 'string "~{~{~2,'0d" delim-string "~}~%~}")
nested-list))
(defun main ()
(let* ((csvfile-path #p"projekte/common-lisp/example_comma_csv.txt")
(result-path #p"results.txt")
(data-list (csvfile-to-nested-list csvfile-path #\,))
(list-of-sums (sublist-sum-list data-list))
(result-header "C1,C2,C3,C4,C5,SUM"))
(setf data-list ; add list of sums as additional column
(rest ; remove old header
(append-each-sublist data-list
(mapcar #'list list-of-sums))))
;; write to output-file
(with-open-file (output result-path :direction :output :if-exists :supersede)
(format output "~a~%~a"
result-header (nested-list-to-csv data-list ",")))))
(main)

View file

@ -0,0 +1,51 @@
defmodule Csv do
defstruct header: "", data: "", separator: ","
def from_file(path) do
[header | data] = path
|> File.stream!
|> Enum.to_list
|> Enum.map(&String.trim/1)
%Csv{ header: header, data: data }
end
def sums_of_rows(csv) do
Enum.map(csv.data, fn (row) -> sum_of_row(row, csv.separator) end)
end
def sum_of_row(row, separator) do
row
|> String.split(separator)
|> Enum.map(&String.to_integer/1)
|> Enum.sum
|> to_string
end
def append_column(csv, column_header, column_data) do
header = append_to_row(csv.header, column_header, csv.separator)
data = [csv.data, column_data]
|> List.zip
|> Enum.map(fn ({ row, value }) ->
append_to_row(row, value, csv.separator)
end)
%Csv{ header: header, data: data }
end
def append_to_row(row, value, separator) do
row <> separator <> value
end
def to_file(csv, path) do
body = Enum.join([csv.header | csv.data], "\n")
File.write(path, body)
end
end
csv = Csv.from_file("in.csv")
csv
|> Csv.append_column("SUM", Csv.sums_of_rows(csv))
|> Csv.to_file("out.csv")

View file

@ -0,0 +1,107 @@
program rowsum
implicit none
character(:), allocatable :: line, name, a(:)
character(20) :: fmt
double precision, allocatable :: v(:)
integer :: n, nrow, ncol, i
call get_command_argument(1, length=n)
allocate(character(n) :: name)
call get_command_argument(1, name)
open(unit=10, file=name, action="read", form="formatted", access="stream")
deallocate(name)
call get_command_argument(2, length=n)
allocate(character(n) :: name)
call get_command_argument(2, name)
open(unit=11, file=name, action="write", form="formatted", access="stream")
deallocate(name)
nrow = 0
ncol = 0
do while (readline(10, line))
nrow = nrow + 1
call split(line, a)
if (nrow == 1) then
ncol = size(a)
write(11, "(A)", advance="no") line
write(11, "(A)") ",Sum"
allocate(v(ncol + 1))
write(fmt, "('(',G0,'(G0,:,''',A,'''))')") ncol + 1, ","
else
if (size(a) /= ncol) then
print "(A,' ',G0)", "Invalid number of values on row", nrow
stop
end if
do i = 1, ncol
read(a(i), *) v(i)
end do
v(ncol + 1) = sum(v(1:ncol))
write(11, fmt) v
end if
end do
close(10)
close(11)
contains
function readline(unit, line)
use iso_fortran_env
logical :: readline
integer :: unit, ios, n
character(:), allocatable :: line
character(10) :: buffer
line = ""
readline = .false.
do
read(unit, "(A)", advance="no", size=n, iostat=ios) buffer
if (ios == iostat_end) return
readline = .true.
line = line // buffer(1:n)
if (ios == iostat_eor) return
end do
end function
subroutine split(line, array, separator)
character(*) line
character(:), allocatable :: array(:)
character, optional :: separator
character :: sep
integer :: n, m, p, i, k
if (present(separator)) then
sep = separator
else
sep = ","
end if
n = len(line)
m = 0
p = 1
k = 1
do i = 1, n
if (line(i:i) == sep) then
p = p + 1
m = max(m, i - k)
k = i + 1
end if
end do
m = max(m, n - k + 1)
if (allocated(array)) deallocate(array)
allocate(character(m) :: array(p))
p = 1
k = 1
do i = 1, n
if (line(i:i) == sep) then
array(p) = line(k:i-1)
p = p + 1
k = i + 1
end if
end do
array(p) = line(k:n)
end subroutine
end program

View file

@ -0,0 +1,20 @@
Copies a file with 5 comma-separated values to a line, appending a column holding their sum.
INTEGER N !Instead of littering the source with "5"
PARAMETER (N = 5) !Provide some provenance.
CHARACTER*6 HEAD(N) !A perfect size?
INTEGER X(N) !Integers suffice.
INTEGER LINPR,IN !I/O unit numbers.
LINPR = 6 !Standard output via this unit number.
IN = 10 !Some unit number for the input file.
OPEN (IN,FILE="CSVtest.csv",STATUS="OLD",ACTION="READ") !For formatted input.
READ (IN,*) HEAD !The first line has texts as column headings.
WRITE (LINPR,1) (TRIM(HEAD(I)), I = 1,N),"Sum" !Append a "Sum" column.
1 FORMAT (666(A:",")) !The : sez "stop if no list element awaits".
2 READ (IN,*,END = 10) X !Read a line's worth of numbers, separated by commas or spaces.
WRITE (LINPR,3) X,SUM(X) !Write, with a total appended.
3 FORMAT (666(I0:",")) !I0 editing uses only as many columns as are needed.
GO TO 2 !Do it again.
10 CLOSE (IN) !All done.
END !That's all.

View file

@ -0,0 +1,49 @@
{-# LANGUAGE FlexibleContexts,
TypeFamilies,
NoMonomorphismRestriction #-}
import Data.List (intercalate)
import Data.List.Split (splitOn)
import Lens.Micro
(<$$>) :: (Functor f1, Functor f2) =>
(a -> b) -> f1 (f2 a) -> f1 (f2 b)
(<$$>) = fmap . fmap
------------------------------------------------------------
-- reading and writing
newtype CSV = CSV { values :: [[String]] }
readCSV :: String -> CSV
readCSV = CSV . (splitOn "," <$$> lines)
instance Show CSV where
show = unlines . map (intercalate ",") . values
------------------------------------------------------------
-- construction and combination
mkColumn, mkRow :: [String] -> CSV
(<||>), (<==>) :: CSV -> CSV -> CSV
mkColumn lst = CSV $ sequence [lst]
mkRow lst = CSV [lst]
CSV t1 <||> CSV t2 = CSV $ zipWith (++) t1 t2
CSV t1 <==> CSV t2 = CSV $ t1 ++ t2
------------------------------------------------------------
-- access and modification via lenses
table = lens values (\csv t -> csv {values = t})
row i = table . ix i . traverse
col i = table . traverse . ix i
item i j = table . ix i . ix j
------------------------------------------------------------
sample = readCSV "C1, C2, C3, C4, C5\n\
\1, 5, 9, 13, 17\n\
\2, 6, 10, 14, 18\n\
\3, 7, 11, 15, 19\n\
\4, 8, 12, 16, 20"

View file

@ -0,0 +1,2 @@
sampleSum = sample <||> (mkRow ["SUM"] <==> mkColumn sums)
where sums = map (show . sum) (read <$$> drop 1 (values sample))

View file

@ -0,0 +1,76 @@
(function () {
'use strict';
// splitRegex :: Regex -> String -> [String]
function splitRegex(rgx, s) {
return s.split(rgx);
}
// lines :: String -> [String]
function lines(s) {
return s.split(/[\r\n]/);
}
// unlines :: [String] -> String
function unlines(xs) {
return xs.join('\n');
}
// macOS JavaScript for Automation version of readFile.
// Other JS contexts will need a different definition of this function,
// and some may have no access to the local file system at all.
// readFile :: FilePath -> maybe String
function readFile(strPath) {
var error = $(),
str = ObjC.unwrap(
$.NSString.stringWithContentsOfFileEncodingError(
$(strPath)
.stringByStandardizingPath,
$.NSUTF8StringEncoding,
error
)
);
return error.code ? error.localizedDescription : str;
}
// macOS JavaScript for Automation version of writeFile.
// Other JS contexts will need a different definition of this function,
// and some may have no access to the local file system at all.
// writeFile :: FilePath -> String -> IO ()
function writeFile(strPath, strText) {
$.NSString.alloc.initWithUTF8String(strText)
.writeToFileAtomicallyEncodingError(
$(strPath)
.stringByStandardizingPath, false,
$.NSUTF8StringEncoding, null
);
}
// EXAMPLE - appending a SUM column
var delimCSV = /,\s*/g;
var strSummed = unlines(
lines(readFile('~/csvSample.txt'))
.map(function (x, i) {
var xs = x ? splitRegex(delimCSV, x) : [];
return (xs.length ? xs.concat(
// 'SUM' appended to first line, others summed.
i > 0 ? xs.reduce(
function (a, b) {
return a + parseInt(b, 10);
}, 0
).toString() : 'SUM'
) : []).join(',');
})
);
return (
writeFile('~/csvSampleSummed.txt', strSummed),
strSummed
);
})();

View file

@ -0,0 +1,18 @@
\\ CSV data manipulation
\\ 10/24/16 aev
\\ processCsv(fn): Where fn is an input path and file name (but no actual extension).
processCsv(fn)=
{my(F, ifn=Str(fn,".csv"), ofn=Str(fn,"r.csv"), cn=",SUM",nf,nc,Vr,svr);
if(fn=="", return(-1));
F=readstr(ifn); nf=#F;
F[1] = Str(F[1],cn);
for(i=2, nf,
Vr=stok(F[i],","); if(i==2,nc=#Vr);
svr=sum(k=1,nc,eval(Vr[k]));
F[i] = Str(F[i],",",svr);
);\\fend i
for(j=1, nf, write(ofn,F[j]))
}
\\ Testing:
processCsv("c:\\pariData\\test");

View file

@ -0,0 +1,33 @@
## Create a CSV file
@"
C1,C2,C3,C4,C5
1,5,9,13,17
2,6,10,14,18
3,7,11,15,19
4,8,12,16,20
"@ -split "`r`n" | Out-File -FilePath .\Temp.csv -Force
## Import each line of the CSV file into an array of PowerShell objects
$records = Import-Csv -Path .\Temp.csv
## Sum the values of the properties of each object
$sums = $records | ForEach-Object {
[int]$sum = 0
foreach ($field in $_.PSObject.Properties.Name)
{
$sum += $_.$field
}
$sum
}
## Add a column (Sum) and its value to each object in the array
$records = for ($i = 0; $i -lt $sums.Count; $i++)
{
$records[$i] | Select-Object *,@{Name='Sum';Expression={$sums[$i]}}
}
## Export the array of modified objects to the CSV file
$records | Export-Csv -Path .\Temp.csv -Force
## Display the object in tabular form
$records | Format-Table -AutoSize

View file

@ -0,0 +1,51 @@
EnableExplicit
#Separator$ = ","
Define fInput$ = "input.csv"; insert path to input file
Define fOutput$ = "output.csv"; insert path to output file
Define header$, row$, field$
Define nbColumns, sum, i
If OpenConsole()
If Not ReadFile(0, fInput$)
PrintN("Error opening input file")
Goto Finish
EndIf
If Not CreateFile(1, fOutput$)
PrintN("Error creating output file")
CloseFile(0)
Goto Finish
EndIf
; Read header row
header$ = ReadString(0)
; Determine number of columns
nbColumns = CountString(header$, ",") + 1
; Change header row
header$ + #Separator$ + "SUM"
; Write to output file
WriteStringN(1, header$)
; Read remaining rows, process and write to output file
While Not Eof(0)
row$ = ReadString(0)
sum = 0
For i = 1 To nbColumns
field$ = StringField(row$, i, #Separator$)
sum + Val(field$)
Next
row$ + #Separator$ + sum
WriteStringN(1, row$)
Wend
CloseFile(0)
CloseFile(1)
Finish:
PrintN("")
PrintN("Press any key to close the console")
Repeat: Delay(10) : Until Inkey() <> ""
CloseConsole()
EndIf

View file

@ -0,0 +1,15 @@
data _null_;
infile datalines dlm="," firstobs=2;
file "output.csv" dlm=",";
input c1-c5;
if _n_=1 then put "C1,C2,C3,C4,C5,Sum";
s=sum(of c1-c5);
put c1-c5 s;
datalines;
C1,C2,C3,C4,C5
1,5,9,13,17
2,6,10,14,18
3,7,11,15,19
4,8,12,16,20
;
run;

View file

@ -0,0 +1,6 @@
set f [open example.csv r]
puts "[gets $f],SUM"
while { [gets $f row] > 0 } {
puts "$row,[expr [string map {, +} $row]]"
}
close $f

View file

@ -0,0 +1,7 @@
Sub ReadCSV()
Workbooks.Open Filename:="L:\a\input.csv"
Range("F1").Value = "Sum"
Range("F2:F5").Formula = "=SUM(A2:E2)"
ActiveWorkbook.SaveAs Filename:="L:\a\output.csv", FileFormat:=xlCSV
ActiveWindow.Close
End Sub