Compare commits

...

No commits in common. "master" and "main" have entirely different histories.
master ... main

33 changed files with 171 additions and 1471 deletions

6
.gitignore vendored
View file

@ -1,6 +0,0 @@
*.csv
*.pdftables
*.pdf
__pycache__
secrets.py
.vscode

View file

@ -1,15 +0,0 @@
MAKEFILES = $(wildcard */Makefile)
SUBDIRS = $(MAKEFILES:%/Makefile=%)
all: build check
build: $(SUBDIRS:%=build-%)
check: $(SUBDIRS:%=check-%)
build-%:
$(MAKE) -C $*
check-%:
! grep ^test: $*/Makefile || $(MAKE) -C $* test
! grep ^check: $*/Makefile || $(MAKE) -C $* check

5
README.md Normal file
View file

@ -0,0 +1,5 @@
# mqtt_rewrite
Make SolaX inverters work with Mosquitto by fixing a bug in the SolaX MQTT implementation.
See https://juju.nz/michaelh/post/2021/solax/ for details.

View file

@ -1,151 +0,0 @@
include <threadlib/threadlib.scad>;
$fn = 100;
// https://www.aliexpress.com/item/1005005144919264.html?spm=a2g0o.order_list.order_list_main.5.d8bc1802v8boh9
charger_r1 = 37 / 2; // Top radius.
charger_r2 = 28.8 / 2; // Body outer thread radius.
charger_thread = 1; // Thread depth.
charger_r3 = charger_r2 - charger_thread;
charger_r4 = 37/2; // Nut radius.
charger_pitch = 13.9/9; // Thread ptich.
charger_h1 = 3; // Cap height.
charger_h2 = 36.3; // Body height minus cap.
lugs_s = [6.3, 11, 10];
charger_s = [charger_r1, charger_r1, charger_h1 + charger_h2];
// https://www.gobilda.com/xt60-connector-pack-mh-fc-x-5-fh-mc-x-5/
xt60_s = [ 15.8, 8.1, 16 ];
xt60_wall = 0.6; // Inner wall thickness.
xt60_h1 = 10.7; // Inner cutout depth.
xt60_i = 2.6;
wall = 1.3;
tol = 0.25;
e = 0.004;
e_z = [ 0, 0, e ];
inf = 100;
inf_z = [ 0, 0, inf ];
module xt60_base(x, y, h) {
i = xt60_i * x / xt60_s.x;
translate([-x/2, -y/2, 0])
linear_extrude(h)
polygon([[0, 0],
[x-i, 0],
[x, i],
[x, y-i],
[x-i, y],
[0, y]]);
}
module xt60() {
w = xt60_wall;
difference() {
xt60_base(xt60_s.x, xt60_s.y, xt60_s.z);
translate([w,w,-e])
xt60_base(xt60_s.x-w*2, xt60_s.y-w*2, xt60_h1);
}
}
module cap(r, h, inset) {
// cylinder(r=charger_r1, h=charger_h1);
rotate_extrude() {
polygon([
[0, 0],
[r, 0],
[r, h-inset],
[r-inset, h],
[0, h],
]);
}
}
module charger() {
translate([0, 0, lugs_s.z]) {
translate([0, 0, charger_h2])
cap(charger_r1, charger_h1, 1.5);
cylinder(r=charger_r3, h=charger_h2);
translate([0, 0, 5])
cylinder(r=charger_r2, h=charger_h2-5);
}
translate([-lugs_s.x/2, -lugs_s.y/2, 0]) {
cube([lugs_s.x, 0.3, lugs_s.z]);
translate([0, lugs_s.y, 0])
cube([lugs_s.x, 0.3, lugs_s.z]);
}
}
module upcap(r, h, inset) {
translate([0, 0, h])
mirror([0, 0, 1])
cap(r,h,inset);
}
module body() {
w = charger_r4;
difference() {
upcap(w+wall, charger_h2+lugs_s.z+10, 1.5);
translate([0, 0, wall])
upcap(w+wall/2, charger_h2*2, 1.5);
xt60_hole();
}
difference() {
xt60_sheath();
translate(-e_z)
xt60_hole();
}
}
module top_cap() {
h = 15;
w = charger_r4+wall+tol/2;
difference() {
union() {
translate([0, 0, h/2])
cap(w+wall, h/2, 1.5);
upcap(w+wall, h/2, 1);
}
translate([0, 0, -wall])
cap(w, h, 1.5);
cylinder(r=charger_r2+tol, h=inf);
cut = 20;
translate([(charger_r2+charger_r1+wall)/2+cut/2, 0, 0])
cylinder(r=cut/2, h=30);
translate([-(charger_r2+charger_r1+wall)/2-cut/2, 0, 0])
cylinder(r=cut/2, h=30);
}
}
module xt60_sheath() {
xt60_base(xt60_s.x+wall*2, xt60_s.y+wall*2, xt60_s.z/2);
}
module xt60_hole() {
xt60_base(xt60_s.x+tol*2, xt60_s.y+tol*2, 20);
}
module test() {
translate([charger_r1*3, 0, xt60_s.z])
charger();
translate([0, charger_r1*3, 0])
xt60();
translate([0, 0, 60])
top_cap();
body();
}
module print() {
body();
// translate([0, 50, 10])
// mirror([0,0,1]) top_cap();
}
//top_cap();
//test();
print();
//body();
//# cube([1, 41.6-wall*2, 1], true);

View file

@ -1,35 +0,0 @@
APP = christmas
DEVICE = attiny85
CLOCK = 16500000
PROGRAMMER = -c signalyzer-lite
SRC = $(wildcard *.cc)
OBJ = $(SRC:%.cc=%.o)
FUSES = -U lfuse:w:0xe2:m -U hfuse:w:0xde:m -U efuse:w:0xfe:m
CROSS_COMPILE = avr-
CC = $(CROSS_COMPILE)gcc
CXX = $(CROSS_COMPILE)g++
CFLAGS = -Os -DF_CPU=$(CLOCK) -mmcu=$(DEVICE) -Wall
CXXFLAGS = $(CFLAGS) -std=gnu++11
AVRDUDE = avrdude $(PROGRAMMER) -p $(DEVICE)
all: $(APP).hex
flash: all
$(AVRDUDE) -U flash:w:$(APP).hex:i
usb-flash: $(APP).hex
sudo $(HOME)/bin/micronucleus --run $<
fuse:
$(AVRDUDE) $(FUSES)
%.elf: $(OBJ)
$(CXX) $(CXXFLAGS) -o $@ $(OBJ)
$(CROSS_COMPILE)size -d $@
%.hex: %.elf
avr-objcopy -j .text -j .data -O ihex $< $@
clean:
rm -f $(APP).hex $(APP).elf $(OBJ) *~

View file

@ -1,188 +0,0 @@
/// Christmas star lights.
///
/// Mainly white, but transitions white -> colour -> white and cycles
/// through colours.
///
/// Michael Hope <michaelh@juju.net.nz> 2013
///
#include <avr/io.h>
#include <avr/interrupt.h>
#include <avr/sleep.h>
class HAL
{
public:
enum Clock {
Timer0Prescaler = 64,
Timer0Rate = F_CPU / Timer0Prescaler / 256,
};
enum class Pin {
};
};
class Christmas
{
public:
static void init();
static void run();
static volatile uint8_t ticks;
private:
/// A single point in the cycle.
struct Point
{
/// How long to stay at this point.
uint8_t seconds;
uint8_t r, g, b;
};
static void delay(uint16_t count);
static uint8_t correct(uint8_t v);
static uint8_t project(uint8_t from, uint8_t to, uint8_t at, uint8_t limit);
static const uint8_t ciel8_[];
static const Point points_[];
};
volatile uint8_t Christmas::ticks;
// Output compare comes out of:
// OC0A (PB0)
// OC0B (PB1)
// OC1B (PB4)
void Christmas::init()
{
DDRB = _BV(0) | _BV(1) | _BV(4);
// Clear the output on compare.
TCCR0A = 0
| (3 << COM0A0)
| (3 << COM0B0)
// Fast PWM.
| (3 << WGM00)
;
static_assert(HAL::Timer0Prescaler == 64, "Update the prescaler below.");
TCCR0B = 0
// Prescale by 64.
| (3 << CS00)
;
TIMSK = _BV(TOIE0);
TCCR1 = 0
| (3 << CS10)
;
GTCCR = 0
| (1 << PWM1B)
| (2 << COM1B0)
;
}
/// Convert an intensity to PWM.
const uint8_t Christmas::ciel8_[] = {
0, 0, 0, 0, 0, 1, 1, 1,
1, 1, 1, 2, 2, 2, 2, 3,
3, 3, 4, 4, 5, 5, 6, 7,
7, 8, 9, 10, 11, 12, 13, 14,
16, 17, 19, 21, 23, 25, 27, 30,
33, 36, 39, 43, 47, 51, 56, 61,
67, 73, 80, 88, 96, 105, 115, 125,
137, 149, 163, 178, 195, 213, 233, 255,
};
#define MAX 63
/// All points to cycle through.
const Christmas::Point Christmas::points_[] =
{
{ 60, MAX, MAX, MAX }, // White
{ 10, 0, 0, MAX }, // Blue
{ 60, MAX, MAX, MAX },
{ 10, MAX, 0, MAX }, // Purple
{ 60, MAX, MAX, MAX },
{ 10, MAX, 0, 0 }, // Red
{ 60, MAX, MAX, MAX },
{ 10, MAX, MAX, 0 }, // Yellow
{ 60, MAX, MAX, MAX },
{ 10, 0, MAX, 0 }, // Green
{ 60, MAX, MAX, MAX },
{ 10, 0, MAX, MAX }, // Cyan
// End of list.
{ 0 },
};
/// Convert intensity to PWM.
uint8_t Christmas::correct(uint8_t level)
{
return ciel8_[level];
}
/// Mix the from and to levels.
uint8_t Christmas::project(uint8_t from, uint8_t to, uint8_t at, uint8_t limit)
{
int delta = to - from;
delta = delta * at / limit;
return (uint8_t)(from + delta);
}
/// Delay the given number of ticks.
void Christmas::delay(uint16_t count)
{
uint8_t now = ticks;
while (count != 0) {
while (now == ticks) {
sleep_mode();
}
count--;
now++;
}
}
void Christmas::run()
{
sei();
const Point* plast = points_ + 0;
for (;;) {
for (const Point* ppoint = points_; ppoint->seconds != 0; ppoint++) {
const int steps = 64;
// Transition between colours.
for (int i = 0; i < steps; i++) {
OCR0A = correct(project(plast->r, ppoint->r, i, steps));
OCR0B = correct(project(plast->g, ppoint->g, i, steps));
OCR1B = 255 - correct(project(plast->b, ppoint->b, i, steps));
// Change over 10s.
delay(HAL::Clock::Timer0Rate * 10 / steps);
}
plast = ppoint;
// Delay on this colour.
delay(HAL::Clock::Timer0Rate * ppoint->seconds);
}
}
}
ISR(TIMER0_OVF_vect)
{
Christmas::ticks++;
}
int main()
{
Christmas::init();
Christmas::run();
return 0;
}

View file

@ -1,35 +0,0 @@
APP = chroma
DEVICE = attiny85
CLOCK = 16500000
PROGRAMMER = -c signalyzer-lite
SRC = $(wildcard *.cc)
OBJ = $(SRC:%.cc=%.o)
FUSES = -U lfuse:w:0xe2:m -U hfuse:w:0xde:m -U efuse:w:0xfe:m
CROSS_COMPILE = avr-
CC = $(CROSS_COMPILE)gcc
CXX = $(CROSS_COMPILE)g++
CFLAGS = -Os -DF_CPU=$(CLOCK) -mmcu=$(DEVICE) -Wall
CXXFLAGS = $(CFLAGS) -std=gnu++11
AVRDUDE = avrdude $(PROGRAMMER) -p $(DEVICE)
all: $(APP).hex
flash: all
$(AVRDUDE) -U flash:w:$(APP).hex:i
usb-flash: $(APP).hex
sudo $(HOME)/bin/micronucleus --run $<
fuse:
$(AVRDUDE) $(FUSES)
%.elf: $(OBJ)
$(CXX) $(CXXFLAGS) -o $@ $(OBJ)
$(CROSS_COMPILE)size -d $@
%.hex: %.elf
avr-objcopy -j .text -j .data -O ihex $< $@
clean:
rm -f $(APP).hex $(APP).elf $(OBJ) *~

View file

@ -1,189 +0,0 @@
/// Flash an RGB LED through all colours.
#include <avr/io.h>
#include <avr/interrupt.h>
#include <avr/sleep.h>
#include <stdlib.h>
class HAL
{
public:
enum Clock {
Timer0Prescaler = 64,
Timer0Rate = F_CPU / Timer0Prescaler / 256,
};
enum class Pin {
};
};
class Chroma
{
public:
static void init();
static void run();
static volatile uint8_t ticks;
private:
struct RGB
{
int16_t r, g, b;
};
static void delay(uint8_t count);
static uint8_t clip(int16_t v);
static uint8_t correct(int16_t v);
static void hsv_to_rgb(int16_t hdash, RGB* rgb);
static const int16_t Scale = 256/2;
static const int16_t One = Scale - 1;
static const uint8_t ciel8[];
};
volatile uint8_t Chroma::ticks;
// Output compare comes out of:
// OC0A (PB0)
// OC0B (PB1)
// OC1B (PB4)
void Chroma::init()
{
DDRB = _BV(0) | _BV(1) | _BV(4);
// Clear the output on compare.
TCCR0A = 0
| (3 << COM0A0)
| (3 << COM0B0)
// Fast PWM.
| (3 << WGM00)
;
static_assert(HAL::Timer0Prescaler == 64, "Update the prescaler below.");
TCCR0B = 0
// Prescale by 64.
| (3 << CS00)
;
TIMSK = _BV(TOIE0);
TCCR1 = 0
| (3 << CS10)
;
GTCCR = 0
| (1 << PWM1B)
| (2 << COM1B0)
;
}
/// Convert an intensity to PWM.
const uint8_t Chroma::ciel8[] = {
0, 0, 0, 0, 0, 1, 1, 1,
1, 1, 1, 2, 2, 2, 2, 3,
3, 3, 4, 4, 5, 5, 6, 7,
7, 8, 9, 10, 11, 12, 13, 14,
16, 17, 19, 21, 23, 25, 27, 30,
33, 36, 39, 43, 47, 51, 56, 61,
67, 73, 80, 88, 96, 105, 115, 125,
137, 149, 163, 178, 195, 213, 233, 255,
};
/// Clip a value to 0..1.
uint8_t Chroma::clip(int16_t v)
{
if (v > One) {
return One;
} else if (v < 0) {
return 0;
} else {
return (uint8_t)v;
}
}
/// Clip and correct a intensity.
uint8_t Chroma::correct(int16_t v)
{
return ciel8[clip(v)/(Scale/64)];
}
/// Convert HSV to RGB.
/// hdash is 0..6.
void Chroma::hsv_to_rgb(int16_t hdash, RGB* rgb)
{
const int16_t xs = One;
const int16_t xv = One;
int16_t chroma = xs * xv / Scale;
int16_t x = chroma * (1*Scale - abs((hdash % (Scale*2)) - Scale)) / Scale;
if (hdash < 1*Scale)
{
*rgb = { chroma, x, 0 };
} else if (hdash < 2*Scale) {
*rgb = { x, chroma, 0 };
} else if (hdash < 3*Scale) {
*rgb = { 0, chroma, x };
} else if (hdash < 4*Scale) {
*rgb = { 0, x, chroma };
} else if (hdash < 5*Scale) {
*rgb = { x, 0, chroma };
} else {
*rgb = { chroma, 0, x };
}
int16_t min = xv - chroma;
rgb->r += min;
rgb->g += min;
rgb->b += min;
}
/// Delay the given number of ticks.
void Chroma::delay(uint8_t count)
{
uint8_t now = ticks;
while (count != 0) {
while (now == ticks) {
sleep_mode();
}
count--;
now++;
}
}
void Chroma::run()
{
sei();
for (;;) {
int16_t step = 1;
// Cycle around the hues.
for (int16_t h = 0; h < Scale*(360/60); h += step) {
RGB rgb;
hsv_to_rgb(h, &rgb);
OCR0A = correct(rgb.r);
OCR0B = correct(rgb.g);
OCR1B = 255 - correct(rgb.b);
delay(5);
}
}
}
ISR(TIMER0_OVF_vect)
{
Chroma::ticks++;
}
int main()
{
Chroma::init();
Chroma::run();
return 0;
}

View file

@ -1,13 +0,0 @@
#include <stdint.h>
void serializeint16(int16_t v);
void serializeint8(int8_t v);
#include "make_struct.h"
#include "servo_param.h"
#include "make_serializer.h"
#include "servo_param.h"
#include "make_init.h"
#include "servo_param.h"

View file

@ -1,8 +0,0 @@
#undef NAME
#undef FIELD
#undef END
#define NAME(x) void reset##x(x##_t* value) { \
*value = {
#define FIELD(type, name, def, doc) .name = def,
#define END(x) }; }

View file

@ -1,7 +0,0 @@
#undef NAME
#undef FIELD
#undef END
#define NAME(x) void serialize##x(const x##_t* value) {
#define FIELD(type, name, def, doc) serialize##type(value->name);
#define END(x) }

View file

@ -1,7 +0,0 @@
#undef NAME
#undef FIELD
#undef END
#define NAME(x) typedef struct x##_s {
#define FIELD(type, name, def, doc) type##_t name;
#define END(x) } x##_t;

View file

@ -1,6 +0,0 @@
NAME(servoParam)
FIELD(int16, min, 1000, "Pulse for minimum travel")
FIELD(int16, max, 2000, "Pulse width for maximum travel")
FIELD(int16, middle, 1500, "Pulse width for middle position")
FIELD(int8, rate, 10, "Travel rate in us/s")
END(servoParam)

14
etc/init.d/mqtt_rewrite Executable file
View file

@ -0,0 +1,14 @@
#!/bin/sh /etc/rc.common
# OpenWRT compatible init script.
USE_PROCD=1
START=95
STOP=01
start_service() {
procd_open_instance
procd_set_param command ujail -n mqtt_rewrite -R / -o -U nobody -- /usr/bin/mqtt_rewrite
procd_set_param stdout 1
procd_set_param stderr 1
procd_set_param respawn
procd_close_instance
}

152
mqtt_rewrite Executable file
View file

@ -0,0 +1,152 @@
#!/usr/bin/micropython
"""Accepts MQTT connections and rewrites the PUBLISH messages before forwarding on.
This can be used to fix the invalid packed ID sent by Solax inverters."""
import asyncio
import ssl
import socket
import sys
from errno import EINPROGRESS
try:
import asyncio.core
import asyncio.stream
IS_MICROPYTHON = True
except:
IS_MICROPYTHON = False
try:
from typing import Optional
except:
# MicroPython doesn't have `typing`.
pass
def _log(msg: str) -> None:
print(msg)
def _open_micropython_ssl_connection(host: str, port: int):
"""MicroPython 1.21 implementation of asyncio.open_connection."""
ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)[0]
s = socket.socket(ai[0], ai[1], ai[2])
s.setblocking(False)
try:
s.connect(ai[-1])
except OSError as er:
if er.errno != EINPROGRESS:
raise er
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.verify_mode = ssl.CERT_NONE
s = context.wrap_socket(s, server_hostname=host, do_handshake_on_connect=False)
s.setblocking(False)
ss = asyncio.stream.Stream(s)
yield asyncio.core._io_queue.queue_write(s)
return ss, ss
async def _open_ssl_connection(host: str, port: int):
if IS_MICROPYTHON:
return await _open_micropython_ssl_connection(host, port)
else:
return await asyncio.open_connection(host, port, ssl=True)
async def _try_get(reader, writer) -> Optional[int]:
"""Returns the next byte from `reader`, or None if closed."""
got = await reader.read(1)
if not got:
return None
writer.write(got)
return got[0]
async def _get(reader, writer) -> int:
"""Returns the next byte from `reader`, or Exception on closed."""
got = await _try_get(reader, writer)
if got is None:
raise EOFError("Unexpected EOF")
return got
async def _get_length(reader, writer) -> int:
"""Returns the encoded length, or Exception on closed or oversize."""
lsb = await _get(reader, writer)
if (lsb & 0x80) == 0:
return lsb
msb = await _get(reader, writer)
if (msb & 0x80) != 0:
# Protect the system by faulting on messages greater than ~16 KiB.
raise OSError("Packet is too long")
return (msb << 7) | (lsb & 0x7F)
def _rewrite_publish(message: bytearray) -> None:
"""Rewrites a publish message by fixing the packet offset if invalid."""
topic_length = (message[0] << 8) | message[1]
packet_id_offset = topic_length + 2
if message[packet_id_offset] == 0 and message[packet_id_offset + 1] == 0:
message[packet_id_offset] = message[0]
message[packet_id_offset + 1] = message[1]
async def _rewrite(direction: str, reader, writer) -> None:
"""Copies messages from `reader` to `writer` with rewrites."""
while True:
packet_type = await _try_get(reader, writer)
if packet_type is None:
break
length = await _get_length(reader, writer)
_log(f"{direction} type={packet_type:x} length={length}")
payload = bytearray(await reader.readexactly(length))
qos = (packet_type >> 1) & 0x03
if (packet_type & 0xF0) == 0x30 and qos > 0:
_rewrite_publish(payload)
writer.write(payload)
await writer.drain()
async def _client(reader, writer, upstream: str, upstream_port: int) -> None:
try:
ureader, uwriter = await _open_ssl_connection(upstream, upstream_port)
try:
await asyncio.gather(
asyncio.create_task(_rewrite("<", ureader, writer)),
asyncio.create_task(_rewrite(">", reader, uwriter)),
)
finally:
uwriter.close()
except Exception as ex:
_log(f"Giving up due to exception: {type(ex)} {ex}")
finally:
writer.close()
async def _serve(
listen: str, listen_port: int, upstream: str, upstream_port: int
) -> None:
async def _wrapper(reader, writer):
return await _client(reader, writer, upstream, upstream_port)
server = await asyncio.start_server(_wrapper, listen, listen_port)
async with server:
await server.wait_closed()
def main():
args = sys.argv[1:] + [None] * 4
upstream = args[0] or "localhost"
upstream_port = int(args[1] or 8883)
listen = args[2] or "0.0.0.0"
listen_port = int(args[3] or 2901)
asyncio.run(_serve(listen, listen_port, upstream, upstream_port))
if __name__ == "__main__":
main()

View file

@ -1,22 +0,0 @@
PDFS = $(wildcard *.pdf)
PFS = $(wildcard export*.pf)
CSVS = $(PDFS:%.pdf=%.csv) $(PFS:%.pf=%.csv)
all: $(CSVS) all.csv
all.csv: $(CSVS)
cat $^ |sort > $@
visebpp_%.csv: visebpp_%.pdftables onecsv.py
python3 onecsv.py --out $@ --csv $<
%.csv: %.pf pfcsv.py
python3 pfcsv.py --out $@ --csv $<
502%.csv: 502%.pdftables cembracsv.py
python3 cembracsv.py --out $@ --csv $<
%.pdftables: %.pdf pdf2csv.py
python3 pdf2csv.py --out $@ --pdf $<
.PRECIOUS: %.pdftables

View file

@ -1,79 +0,0 @@
import csv as pcsv
import dataclasses
from typing import Optional, Tuple
import re
import datetime
import click
@dataclasses.dataclass
class Entry:
date: str
account: str
memo: str
location: str
kind: str
amount: float
def parse_date(text: str) -> datetime.date:
d, m, y = (int(x) for x in text.split('.'))
return datetime.date(year=y, month=m, day=d)
def parse_memo(memo: str) -> Tuple[str, str, str]:
location = ''
kind = ''
memo = memo.split('\n')[0]
match = re.match(r'(.+) ([A-Z]{3})$', memo)
if match:
memo = match.group(1)
location = match.group(2)
return memo, location.strip(), kind
def parse(line: tuple) -> Optional[Entry]:
# 19.10.2021,19.10.2021,Nintendo CD598510225 Frankfurt am DEU,,5.60
if len(line) < 4 or not line[-1]:
return None
if not re.match(r'\d+\.\d+\.\d+', line[0]):
return None
date = parse_date(line[0])
memo = line[2]
memo, location, kind = parse_memo(memo)
amount = float(line[-1].replace("'", ""))
return Entry(date=str(date),
account="cembra",
memo=memo,
location=location,
kind=kind,
amount=amount)
@click.command()
@click.option('--out', required=True, type=str)
@click.option('--csv', required=True, type=str)
def extract(out: str, csv: str):
entries = []
with open(csv) as f:
for line in pcsv.reader(f):
entry = parse(line)
if entry:
entries.append(entry)
with open(out, 'w') as f:
writer = pcsv.writer(f)
for entry in entries:
writer.writerow(dataclasses.astuple(entry))
if __name__ == '__main__':
extract()

View file

@ -1,89 +0,0 @@
import csv as pcsv
import dataclasses
from typing import Optional, Tuple
import re
import datetime
import click
@dataclasses.dataclass
class Entry:
date: str
account: str
memo: str
location: str
kind: str
amount: float
def parse_date(text: str) -> datetime.date:
d, m, y = (int(x) for x in text.split('.'))
return datetime.date(year=2000 + y, month=m, day=d)
def parse_memo(memo: str) -> Tuple[str, str, str]:
parts = memo.split('\n')
if len(parts) == 1:
memo, kind = parts[0], ''
else:
memo, kind = parts[0], parts[1]
parts = memo.rsplit(',', maxsplit=1)
if len(parts) == 1:
memo, location = parts[0], ''
else:
memo, location = parts
return memo, location.strip(), kind
def parse(line: tuple) -> Optional[Entry]:
# ['15.07.22 18.07.22 NYA*Arena Cinemas AG, 433050453 CH\nLebensmittel, Spezialgeschäfte', '', '', '', '', '', '4.00']
if len(line) < 4 or not line[-1]:
return None
if not re.match(r'\d+\.\d+\.\d+ \d+', line[0]):
return None
parts = line[0].split(maxsplit=2)
if len(parts) == 3:
date = parse_date(parts[0])
memo = parts[-1]
else:
date = parse_date(parts[0])
memo = line[2]
memo, location, kind = parse_memo(memo)
parts = line[-1].split(' ', maxsplit=1)
amount = float(parts[0].replace("'", ''))
if len(parts) == 2:
assert parts[-1] == '-'
amount = -amount
return Entry(date=str(date),
account="one",
memo=memo,
location=location,
kind=kind,
amount=amount)
@click.command()
@click.option('--out', required=True, type=str)
@click.option('--csv', required=True, type=str)
def extract(out: str, csv: str):
entries = []
with open(csv) as f:
for line in pcsv.reader(f):
entry = parse(line)
if entry:
entries.append(entry)
with open(out, 'w') as f:
writer = pcsv.writer(f)
for entry in entries:
writer.writerow(dataclasses.astuple(entry))
if __name__ == '__main__':
extract()

View file

@ -1,29 +0,0 @@
import sys
import click
import requests
import secrets
@click.command()
@click.option('--out', required=True, type=str)
@click.option('--pdf', required=True, type=str)
def extract(out: str, pdf: str):
params = {
'key': secrets.PDFTABLES_KEY,
'format': 'csv',
}
with open(pdf, 'rb') as f:
files = {
'f': ('file.pdf', f.read()),
}
body = requests.post('https://pdftables.com/api',
params=params,
files=files)
with open(out, 'w') as f:
f.write(body.text)
if __name__ == '__main__':
extract()

View file

@ -1,127 +0,0 @@
import csv as pcsv
import dataclasses
from typing import Optional, Tuple
import re
import datetime
import click
@dataclasses.dataclass
class Entry:
date: str
account: str
memo: str
location: str
kind: str
amount: float
def parse_dict(filename: str):
words = {}
with open(filename) as f:
for line in f:
for word in line.strip().split():
words[word.lower()] = word
for word in ('AG', 'TWINT', 'SBB'):
words[word.lower()] = word
return words
WORDS = parse_dict('/usr/share/dict/words')
def fix_case(line: str) -> str:
words = []
for i, word in enumerate(line.split()):
word = WORDS.get(word.lower(), word.lower())
if i == 0:
word = word[0].upper() + word[1:]
words.append(word)
return ' '.join(words)
def parse_date(text: str) -> datetime.date:
d, m, y = (int(x) for x in text.split('.'))
return datetime.date(year=y, month=m, day=d)
def parse_memo(memo: str) -> Tuple[str, str, str]:
location = ''
kind = ''
memo = memo.strip()
match = re.match(r'^.+CARD NO. XX\S+\s+(.+)', memo)
if match:
memo = match.group(1)
match = re.match(r'TWINT PURCHASE/SERVICE FROM \S+ FROM TELEPHONE NO. \S+\s+(.+)', memo)
if match:
memo = match.group(1)
match = re.match(r'(DEBIT|CREDIT) CH\d+\s+(.+)', memo)
if match:
memo = match.group(2)
memo = fix_case(memo)
match = re.match(r'(.+) (\w+) (Switzerland|France)$', memo)
if match:
memo, city, country = match.groups()
location = f'{city} {country}'
return memo, location, kind
def parse(line: tuple) -> Optional[Entry]:
# 13.07.2024;Entry;"TWINT PURCHASE/SERVICE FROM 12.07.2024 FROM TELEPHONE NO. +41792456887 MCDONALD'S ZÜRICH SIHLCITY ZUERICH";;-9.50;;
if len(line) < 5 or not line[-1]:
return None
if not re.match(r'\d+\.\d+\.\d+', line[0]):
return None
date = parse_date(line[0])
memo = line[2]
memo, location, kind = parse_memo(memo)
if line[3]:
amount = float(line[3].replace("'", ""))
else:
amount = -float(line[4].replace("'", ""))
if not kind:
kind = line[-1]
return Entry(date=str(date),
account="pf",
memo=memo,
location=location,
kind=kind,
amount=amount)
@click.command()
@click.option('--out', required=True, type=str)
@click.option('--csv', required=True, type=str)
def extract(out: str, csv: str):
entries = []
with open(csv) as f:
dialect = pcsv.Sniffer().sniff(f.read(1024), ';')
f.seek(0)
for line in pcsv.reader(f, dialect=dialect):
entry = parse(line)
if entry:
entries.append(entry)
with open(out, 'w') as f:
writer = pcsv.writer(f)
for entry in entries:
writer.writerow(dataclasses.astuple(entry))
if __name__ == '__main__':
extract()

Binary file not shown.

Before

Width:  |  Height:  |  Size: 434 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 784 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 296 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 263 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 200 B

View file

@ -1,210 +0,0 @@
"""Converts a bitmap image into a minimal number of covering
rectangles.
Mainly plays with different methods.
"""
# python painter.py images/*.png
from __future__ import print_function
import sys
import collections
import time
import numpy
import scipy.misc
# A rectangle.
Rect = collections.namedtuple('Rect', 'x y w h')
# Chracters to use when printing the image to the screen.
EXT = range(ord('1'), ord('9') + 1) \
+ range(ord('a'), ord('z') + 1) \
+ range(ord('A'), ord('Z') + 1)
CHARS = [ord('.'), ord('#')] + EXT*20
def parse(filename):
"""Parse a image into a (0, 1) numpy array."""
img = scipy.misc.imread(filename)
img = numpy.clip(img, 0, 1)
return img ^ 1
def render(shape, rects, with_ids=False):
"""Render rectangles into a new array of the given shape."""
img = numpy.zeros(shape, numpy.uint16)
for i, rect in enumerate(rects):
for y in range(rect.y, rect.y + rect.h):
for x in range(rect.x, rect.x + rect.w):
if with_ids:
img[y][x] = i + 2
else:
img[y][x] = 1
return img
def show(img):
"""Print an image to the console."""
for row in img:
print(''.join(chr(CHARS[x]) for x in row))
def pointwise(img):
"""Method where each point is a rectangle."""
for y, row in enumerate(img):
for x, pix in enumerate(row):
if pix:
yield Rect(x, y, 1, 1)
def rowwise(img):
"""Method where rectangles are one row high."""
for y, row in enumerate(img):
start = None
for x, pix in enumerate(row):
if pix:
if start is None:
start = x
else:
if start is not None:
yield Rect(start, y, x - start, 1)
start = None
if start is not None:
yield Rect(start, y, x - start + 1, 1)
def rotate(img, fun):
"""Rotate an image and rotate the final rectangles."""
h, w = img.shape
for rect in fun(numpy.rot90(img)):
yield Rect(w - rect.y - 1, rect.x, rect.h, rect.w)
def colwise(img):
"""Method where rectangles are one column wide."""
return list(rotate(img, rowwise))
def mergeup(rects):
"""Merge a rectangle into the identical width one below it."""
# TODO(michaelh): ugly.
while True:
out = []
merged = set()
rects.sort(key=lambda x: x.y)
for rect in rects:
if rect in merged:
continue
top = rect.y + rect.h
below = [x for x in rects if x.y == top]
matches = [x for x in below if x.x == rect.x and x.w == rect.w]
if matches:
match = matches[0]
merged.add(match)
out.append(Rect(rect.x, rect.y, rect.w, rect.h + match.h))
else:
out.append(rect)
if not merged:
return out
rects = out
def mergeleft(rects):
"""Merge a rectangle into the identical height one beside it."""
# TODO(michaelh): merge with mergeup().
while True:
out = []
merged = set()
rects.sort(key=lambda x: x.x)
for rect in rects:
if rect in merged:
continue
edge = rect.x + rect.w
adjacent = [x for x in rects if x.x == edge]
matches = [x for x in adjacent if x.y == rect.y and x.h == rect.h]
if matches:
match = matches[0]
merged.add(match)
out.append(Rect(rect.x, rect.y, rect.w + match.w, rect.h))
else:
out.append(rect)
if not merged:
return out
rects = out
def rowcolwise(img):
"""Method that picks the best of rowwise and colwise."""
rows = list(rowwise(img))
cols = list(colwise(img))
return rows if len(rows) < len(cols) else cols
def mergerowcolwise(img):
"""Method that picks the best of the merged rowwise and colwise."""
rows = mergeup(list(rowwise(img)))
cols = mergeleft(list(colwise(img)))
return rows if len(rows) < len(cols) else cols
def bothwise(img):
"""Method that combines row and col wise, then drops rectangles
that are completly covered by larger rectangles.
"""
rows = list(rowwise(img))
cols = list(colwise(img))
rects = rows + cols
# Sort by area. Smaller are less likely to contribute.
rects.sort(key=lambda x: x.w*x.h)
drop = set()
for rect in rects:
others = [x for x in rects if x != rect and x not in drop]
got = render(img.shape, others)
if not (got ^ img).any():
# Dropping this rect didn't change the image.
drop.add(rect)
rects = [x for x in rects if x not in drop]
return mergeleft(mergeup(rects))
def main():
# All methods to try.
methods = [
pointwise,
rowwise,
colwise,
rowcolwise,
mergerowcolwise,
bothwise,
]
for filename in sys.argv[1:]:
img = parse(filename)
best = None
print('{}:'.format(filename))
for method in methods:
start = time.clock()
scan = list(method(img))
elapsed = time.clock() - start
name = method.__name__
print(' {} -> {} rects in {:.3f} s'.format(name, len(scan), elapsed))
rendered = render(img.shape, scan)
diff = img ^ rendered
if diff.any():
print('Render error!')
else:
if best is None or len(scan) <= len(best):
best = scan
# Show the best.
show(render(img.shape, best, True))
if __name__ == '__main__':
main()

View file

@ -1,10 +0,0 @@
# http://kjell.haxx.se/sudoku/
......635
.7.56....
3.....84.
..327.1.6
.........
52.8.6.9.
.61...5..
......4..
...3.4.6.

View file

@ -1,10 +0,0 @@
# http://kjell.haxx.se/sudoku/
.5.4.....
7.....28.
.......1.
....28...
....1....
.4.....9.
8.......7
1...7....
...3....5

View file

@ -1,9 +0,0 @@
CXXFLAGS = -O3 -std=c++0x
BOARDS = $(wildcard *.board)
all: sudoku
check: $(BOARDS:%.board=%.run)
%.run: %.board sudoku
./sudoku < $<

View file

@ -1,14 +0,0 @@
Sudoku Solver
=============
Done for the Olimex Weekend Programming Challenge #28 [1].
Build: make
Run: make test
Benchmarks:
Feroceon 88FR131 / 2.0 GHz ARMv5 / GCC 4.6.3: 16.0 M/s
i.MX6 Quad / 1.0 GHz Cortex-A9 / GCC 4.7.3: 14.4 M/s
Core 2 P8600 / 2.4 GHz Core 2 / GCC 4.7.3: 54.2 M/s
[1] http://olimex.wordpress.com/2013/10/11/weekend-programming-challenge-week-28-sudoku-solver/

View file

@ -1,9 +0,0 @@
.....8..4
.84.16...
...5..1..
1.38..9..
6.8...4.3
..2..95.1
..7..2...
...78.26.
2..3.....

View file

@ -1,11 +0,0 @@
# "World's hardest sudoku: can you crack it?"
# http://www.telegraph.co.uk/science/science-news/9359579/Worlds-hardest-sudoku-can-you-crack-it.html
8........
..36.....
.7..9.2..
.5...7...
....457..
...1...3.
..1....68
..85...1.
.9....4..

View file

@ -1,192 +0,0 @@
// Sudoku solver
// MIT license michaelh@juju.net.nz
#include <cstdio>
#include <cstring>
#include <cctype>
#include <cassert>
#include <ctime>
#include <cstdint>
// Uses sets to find out what's used and what the possibilities are.
// Implemented as a bitmask so an empty cell has the value 0, a cell
// with a 3 in contains 1 << 3, etc.
class Board
{
public:
uint16_t cells[9][9];
void Read();
void Print() const;
};
struct CellRef
{
int x;
int y;
};
class Stats
{
public:
int solutions;
void Init();
void Print();
void Bump();
private:
clock_t start;
uint64_t tests;
};
Stats stats;
static int FFS(int v)
{
return sizeof(v)*8 - 1 - __builtin_clz(v);
}
void Stats::Init()
{
solutions = 0;
tests = 0;
start = clock();
}
void Stats::Print()
{
double elapsed = (clock() - stats.start) / double(CLOCKS_PER_SEC);
printf("%llu tests %d solutions %.3f s (%.1f M/s)\n",
stats.tests, stats.solutions,
elapsed, stats.tests / elapsed / 1e6);
}
void Stats::Bump()
{
stats.tests++;
if ((stats.tests & 0x3ffffff) == 0) {
stats.Print();
}
}
void Board::Read()
{
int x = 0;
int y = 0;
int ch;
memset(cells, 0, sizeof(cells));
while ((ch = fgetc(stdin)) != EOF) {
if (ch == '#') {
// Drop comment lines (kind of)
while ((ch = fgetc(stdin)) != EOF && ch != '\n') {
}
} else if (ch == '\n') {
x = 0;
y++;
} else if (ch == '\r') {
// Drop
} else if (ch == '.') {
cells[y][x++] = 0;
} else if (isdigit(ch)) {
cells[y][x++] = 1 << (ch - '0');
} else {
assert(false);
}
}
}
void Board::Print() const
{
for (int y = 0; y < 9; y++) {
for (int x = 0; x < 9; x++) {
if (cells[y][x] == 0) {
putchar('.');
} else {
putchar('0'+ FFS(cells[y][x]));
}
}
printf("\n");
}
printf("\n");
}
// Find all of the blank cells. Terminate the list with -1.
static void FindBlanks(const Board& board, CellRef* prefs)
{
for (int y = 0; y < 9; y++) {
for (int x = 0; x < 9; x++) {
if (board.cells[y][x] == 0) {
prefs->x = x;
prefs->y = y;
prefs++;
}
}
}
prefs->x = -1;
}
static void OnSolution(const Board& board)
{
stats.solutions++;
// Print the first solution and keep going.
if (stats.solutions == 1) {
board.Print();
}
}
static void Solve(Board& board, const CellRef* pref)
{
// Mask for 1 to 9
const int All = (1 << 10) - 1 - 1;
int x = pref->x;
int y = pref->y;
if (x == -1) {
OnSolution(board);
}
else {
int present = 0;
// Scan across the row and down the column to see which digits
// are already used.
for (int i = 0; i < 9; i++) {
present |= board.cells[y][i];
present |= board.cells[i][x];
}
// Calculate the unused values.
int possibles = (present & All) ^ All;
while (possibles != 0) {
// Find the next and mark it as done.
int next = 1 << FFS(possibles);
possibles &= ~next;
stats.Bump();
// Solve for the next blank cell.
board.cells[y][x] = next;
Solve(board, pref+1);
board.cells[y][x] = 0;
}
}
}
int main()
{
Board board;
CellRef refs[100];
board.Read();
board.Print();
stats.Init();
FindBlanks(board, refs);
Solve(board, refs);
stats.Print();
return 0;
}