Commit 41342b78 authored by Charles Bouillaguet's avatar Charles Bouillaguet
Browse files

more code

parent c6a5a210
......@@ -11,3 +11,4 @@ moebius_demo
/macaulay_gen
/lanczos
venv
/reconstruct_monomials
......@@ -10,6 +10,7 @@ import mqlogger
import logging
import pathlib
import math
import socket
try:
import yaml
......@@ -24,7 +25,7 @@ class Parameters:
inner_hybridation = 0
degree = 3
linear_variables = 7
options = {'linear_algebra': {'software': 'lanczos'}}
def __init__(self, **kwds):
for k, v in kwds.items():
setattr(self, k, v)
......@@ -89,7 +90,7 @@ class Main:
best = pset
mbest, nbest = best['m'], best['n']
if m > mbest or (m == mbest and n > nbest):
best = (m, n)
best = pset
if best is None:
logger.warning("No suitable parameter sets. Auto-choosing fail-safe (bad) parameters")
......@@ -103,57 +104,141 @@ class Main:
logger.warning(f'Auto-choosing parameters (no precise match). Using m={mbest}, n={nbest}')
return Parameters(**best)
def run(self, name, cmd):
"""
Execute external command
"""
args = self.args
logger = logging.getLogger(name)
logger = self.logger
logger.command(' '.join(cmd))
if args.dry_run:
return
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
with process.stdout:
for line in iter(process.stdout.readline, b''): # b'\n'-separated lines
logger.info(line.decode().strip())
exitcode = process.wait() # 0 means success
if exitcode != 0:
raise ValueError("Something failed")
try:
logger = logging.getLogger(name)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
with process.stdout:
for line in iter(process.stdout.readline, b''): # b'\n'-separated lines
logger.info(line.decode().strip())
exitcode = process.wait() # 0 means success
if exitcode != 0:
raise ValueError("Something failed")
except KeyboardInterrupt:
logger = self.logger
logger.fatal("Received KeyboardInterrupt. Terminating")
logger.info("Resume with %s --workdir %s %s", sys.argv[0], self.wdir, self.args.filename)
sys.exit(1)
def macaulay_step(self, filename, output_base):
def macaulay_step(self, filename):
parameters = self.parameters
cmd = ['./macaulay_gen', '--in', filename, '--degree', str(parameters.degree),
'--inner-hybridation', str(parameters.linear_variables), '--out', str(output_base)]
self.run('macaulay_gen', cmd)
wdir = self.wdir
D = {}
D['matrix'] = str(output_base) + '.bin'
D['ilog'] = str(output_base) + '.ilog.bin'
D['jlog'] = str(output_base) + '.jlog.bin'
D['matrix'] = wdir / 'matrix.bin'
D['ilog'] = wdir / 'matrix.ilog.bin'
D['jlog'] = wdir / 'matrix.jlog.bin'
cmd = ['./macaulay_gen', '--in', filename, '--degree', str(parameters.degree),
'--inner-hybridation', str(parameters.linear_variables), '--out', str(wdir / 'matrix')]
# should we run?
if os.path.exists(D['jlog']):
self.logger.command('macaulay matrix already present')
else:
self.run('macaulay_gen', cmd)
return D
def linear_algebra_step(self, Dma, output):
def cadopath(self):
"""
locate cado binaries
"""
hostname = socket.gethostname()
logger = self.logger
if self.args.cado_build_dir:
base = pathlib.Path(self.args.cado_build_dir)
else:
# try in the current directory?
logger.debug("No --cado-build-dir given. Searching in ./cado-nfs")
base = pathlib.Path('./cado-nfs')
if not os.path.exists(base):
raise ValueError('--cado-build-dir not given. Where is CADO-NFS?')
logger.debug("Found something in ./cado-nfs")
if not os.path.exists(base / 'build'):
raise ValueError("CADO-NFS source dir, but no build dir. Run ``make'' in the cado-nfs/ directory")
if not os.path.exists(base / hostname):
raise ValueError("CADO-NFS source dir, but no binaries for this host. Run ``make'' in the cado-nfs/ directory")
target = base / hostname / 'linalg' / 'bwc'
if not os.path.exists(target):
raise ValueError(f'Bizarre. Cannot find bwc binaries')
return target
def linear_algebra_step(self, Dma):
wdir = self.wdir
D = {'kernel': wdir / 'kernel.bin'}
parameters = self.parameters
cmd = ['./lanczos', '--matrix', Dma['matrix'], '--output', str(output)]
self.run('lanczos', cmd)
return {'kernel': output}
software = parameters.options['linear_algebra']['software']
if software == 'CADO-NFS':
bwc_wdir = pathlib.Path(self.wdir) / 'bwc'
D['kernel'] = bwc_wdir / 'W'
# should we run?
if os.path.exists(D['kernel']):
self.logger.command('kernel vector already present')
return D
if software == 'lanczos':
cmd = ['./lanczos', '--matrix', str(Dma['matrix']), '--output', str(D['kernel'])]
self.run('lanczos', cmd)
elif software == 'CADO-NFS':
cadopath = self.cadopath()
if not os.path.exists(wdir):
os.mkdir(wdir)
bwc = str(cadopath / 'bwc.pl')
cmd = [bwc, ':complete', 'balancing_options=reorder=columns',
'matrix=' + str(Dma['matrix']), 'wdir=' + str(bwc_wdir), 'mn=64']
extra = parameters.options['linear_algebra'].get('bwc_extra_opts', [])
cmd.extend(extra)
self.run('bwc', cmd)
else:
self.logger.error(f"unknown software ``{software}''")
sys.exit(1)
return D
def reconstruct_step(self, filename, Dma, Dla, output):
def reconstruct_step(self, filename, Dma, Dla):
parameters = self.parameters
cmd = ['./reconstruct_monomials', '--in', filename, '--degree', str(parameters.degree),
'--inner-hybridation', str(parameters.linear_variables), '--out', str(output),
'--ilog', Dma['ilog'], '--jlog', Dma['jlog'], '--kernel', str(Dla['kernel'])]
self.run('reconstruct', cmd)
return {'polynomials': output}
wdir = self.wdir
D = {'polynomials': wdir / 'new_system.txt'}
cmd = ['./reconstruct_monomials', '--in', str(filename), '--degree', str(parameters.degree),
'--inner-hybridation', str(parameters.linear_variables), '--out', str(D['polynomials']),
'--ilog', str(Dma['ilog']), '--jlog', str(Dma['jlog']), '--kernel', str(Dla['kernel'])]
# should we run?
if os.path.exists(D['polynomials']):
self.logger.command('reconstructed system already present')
else:
self.run('reconstruct', cmd)
return D
def go(self):
"""
Manage the whole software stack
"""
logger = self.logger
parser = argparse.ArgumentParser()
parser.add_argument("filename", help="file containing the input system")
parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
parser.add_argument("-n", "--dry-run", help="Don't actually run any command; just print them", action="store_true")
parser.add_argument("--workdir", help="WORKDIR is created if it does not exist. If unspecified, a temporary directory is used")
parser.add_argument("--cado-build-dir", help="where to find the CADO-NFS binaries")
self.args = parser.parse_args()
args = self.args
......@@ -165,7 +250,8 @@ class Main:
logfilename = wdir / "mq.log"
filehandler = mqlogger.FileHandler(filename=logfilename, lvl=logging.DEBUG)
logger.addHandler(filehandler)
logger.info("If this computation gets interrupted, it can be resumed with %s --workdir %s", sys.argv[0], wdir)
logger.info("If this computation gets interrupted, it can be resumed with %s --workdir %s %s",
sys.argv[0], wdir, args.filename)
# parse input system
n, m = self.parse_input_system(args.filename)
......@@ -183,10 +269,16 @@ class Main:
e = n - params.outer_hybridation - params.linear_variables
logger.info(f'Enumeration on {e} variables')
Dma = self.macaulay_step(args.filename, wdir / "matrix")
Dla = self.linear_algebra_step(Dma, wdir / 'kernel.bin')
Dre = self.reconstruct_step(args.filename, Dma, Dla, wdir / "new_system.in")
Dma = self.macaulay_step(args.filename)
Dla = self.linear_algebra_step(Dma)
Dre = self.reconstruct_step(args.filename, Dma, Dla)
if __name__ == '__main__':
Main().go()
logging.shutdown()
\ No newline at end of file
x = Main()
try:
x.go()
except Exception as e:
x.logger.exception(e)
sys.exit(1)
finally:
logging.shutdown()
\ No newline at end of file
......@@ -13,6 +13,7 @@
#define _POSIX_C_SOURCE 1
#include <inttypes.h>
#include <stdio.h>
#include <unistd.h> // isatty
#include <stdbool.h>
#include <string.h>
#include <stdlib.h>
......@@ -991,9 +992,11 @@ void verbosity()
char ETA[30];
ctime_r(&end, ETA);
ETA[strlen(ETA) - 1] = 0; // remove the final \n
printf("\rItération %ld / %ld. %.3fs per iteration. ETA: %s", n_iterations, expected_iterations, per_iteration, ETA);
fflush(stdout);
ETA[strlen(ETA) - 1] = 0; // remove the final \n
printf("\rItération %ld / %ld. %.3fs per iteration. ETA: %s", n_iterations, expected_iterations, per_iteration, ETA);
if (!isatty(fileno(stdout)))
printf("\n");
fflush(stdout);
}
void test_transpose()
......
......@@ -442,7 +442,7 @@ int main(int argc, char **argv)
snprintf(ilog_filename, 256, "%s.ilog.bin", out_basename);
snprintf(jlog_filename, 256, "%s.jlog.bin", out_basename);
log(1, " - saving to %s / %s / %s / %s / %s\n", matrix_filename,
log(1, " - saving to %s, %s, %s, %s, %s\n", matrix_filename,
rw_filename, cw_filename, ilog_filename, jlog_filename);
buffer_stream = fopen(matrix_filename, "w");
if (buffer_stream == NULL)
......
......@@ -8,19 +8,30 @@
- m: 48
n: 48
degree: 4
linear_variables: 12
linear_variables: 12 # 15s de BL
# good for n=m=56
- m: 56
n: 54
degree: 4
linear_variables: 13 # 45 s de BL
# good for n=m=64
- m: 64
n: 54
degree: 4
linear_variables: 14
linear_variables: 14 # 1 min de BL
# good for n=m=72 ?????
# good for n=m=72
- m: 72
n: 72
degree: 5
linear_variables: 16
linear_variables: 15
options:
linear_algebra:
software: CADO-NFS
bwc_extra_opts:
- thr=4
# good for n=m=80
- m: 80
......@@ -31,6 +42,14 @@
linear_algebra:
software: CADO-NFS
################### underdetermined systems
# good for m=72, n=108
- m: 72
n: 52
degree: 4
linear_variables: 15 # 1min de BL par job
################### overdetermined systems
- m: 148
......
......@@ -15,6 +15,7 @@ struct input_poly_t *reconstruct(char *filename_ker, char *filename_ilog,
struct input_poly_t *f,
struct monomial_ctx_t *mono)
{
log(0, "Reading kernel vector and metadata\n");
u64 size_ker;
u32 size_ilog, size_jlog;
u64 *ker = load_file_64(filename_ker, &size_ker);
......@@ -39,14 +40,22 @@ struct input_poly_t *reconstruct(char *filename_ker, char *filename_ilog,
for (int i = 0; i < size_mono; i++)
res[i] = 0;
log(0, "Matrix-vector product\n");
double start = wtime();
#pragma omp parallel for
for (u64 k = 0; k < size_ker; k++) {
int j = jlog[k];
int i = ilog[k];
for (int l = 0; l < f[i].nterms; l++) {
int x = monomials_product(j, f[i].terms[l], mono);
#pragma omp atomic update
res[x] ^= ker[k];
}
}
log(0, "- done (%.1fs)\n", wtime() - start);
free(ilog);
free(jlog);
free(ker);
......@@ -154,7 +163,12 @@ int main(int argc, char **argv)
err(1, "Cannot open %s", in_filename);
}
log(0, "Writing output system\n");
double start = wtime();
fprintf(output_file, "# produced by %s\n", argv[0]);
system_to_txt(output_file, new, NB_VEC, sys->variable_name, mono);
log(0, "- done (%.1fs)\n", wtime() - start);
return EXIT_SUCCESS;
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment