hyung-hwan/qse
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

almost finised mpi migration

Browse Source
This commit is contained in:
hyung-hwan 2012-11-01 06:42:38 +00:00
parent acb40dea35
commit 3b9f0ab14e
11 changed files with 228 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
qse/cmd/awk/Makefile.am
View File

@ -3,7 +3,8 @@ AUTOMAKE_OPTIONS = nostdinc
AM_CPPFLAGS = \
-I$(top_builddir)/include \
-I$(top_srcdir)/include \
-I$(includedir)
-I$(includedir) \
-DDEFAULT_MODDIR=\"$(libdir)/qse\"
#####################################################################3
@ -18,4 +19,3 @@ if WCHAR
qseawk_LDADD += $(UNICOWS_LIBS)
endif
endif

3
qse/cmd/awk/Makefile.in
View File

@ -254,7 +254,8 @@ AUTOMAKE_OPTIONS = nostdinc
AM_CPPFLAGS = \
-I$(top_builddir)/include \
-I$(top_srcdir)/include \
-I$(includedir)
-I$(includedir) \
-DDEFAULT_MODDIR=\"$(libdir)/qse\"
qseawk_SOURCES = awk.c
qseawk_LDFLAGS = -L../../lib/awk -L../../lib/cmn -L$(libdir)

91
qse/cmd/awk/awk.c
View File

@ -966,7 +966,7 @@ static int awk_main (int argc, qse_char_t* argv[])
qse_awk_parsestd_t psout;
qse_mmgr_t* mmgr = QSE_MMGR_GETDFL();
memset (&arg, 0, QSE_SIZEOF(arg));
qse_memset (&arg, 0, QSE_SIZEOF(arg));
arg.icf.mmgr = mmgr;
i = comparg (argc, argv, &arg);
@ -1129,9 +1129,88 @@ oops:
return ret;
}
struct mpi_t
{
void* h;
int (*i) (int argc, qse_achar_t* argv[]);
void (*f) (void);
};
typedef struct mpi_t mpi_t;
static void open_mpi (mpi_t* mpi, int argc, qse_achar_t* argv[])
{
lt_dladvise adv;
qse_memset (mpi, 0, QSE_SIZEOF(*mpi));
#if defined(USE_LTDL)
#if defined(QSE_ACHAR_IS_MCHAR)
if (qse_mbscmp (qse_mbsbasename(argv[0]), QSE_MT("qseawkmp")) != 0 &&
qse_mbscmp (qse_mbsbasename(argv[0]), QSE_MT("qseawkmpi")) != 0) return;
#else
if (qse_wcscmp (qse_wcsbasename(argv[0]), QSE_WT("qseawkmp")) != 0 &&
qse_wcscmp (qse_wcsbasename(argv[0]), QSE_WT("qseawkmpi")) != 0) return;
#endif
if (lt_dlinit () != 0) return;
if (lt_dladvise_init (&adv) != 0) goto oops;
/* If i don't set the global option, loading may end up with an error
* like this depending on your MPI library.
*
* symbol lookup error: /usr/lib/openmpi/lib/openmpi/mca_paffinity_linux.so: undefined symbol: mca_base_param_reg_int
*/
if (lt_dladvise_global (&adv) != 0 || lt_dladvise_ext (&adv) != 0)
{
lt_dladvise_destroy (&adv);
goto oops;
}
mpi->h = lt_dlopenadvise (DEFAULT_MODDIR "/libawkmpi", adv);
lt_dladvise_destroy (&adv);
if (mpi->h)
{
mpi->i = lt_dlsym (mpi->h, "mpi_init");
mpi->f = lt_dlsym (mpi->h, "mpi_fini");
if (mpi->i == QSE_NULL ||
mpi->f == QSE_NULL ||
mpi->i (argc, argv) <= -1)
{
lt_dlclose (mpi->h);
mpi->h = QSE_NULL;
}
}
return;
oops:
lt_dlexit ();
#endif
}
static void close_mpi (mpi_t* mpi)
{
if (mpi->h)
{
#if defined(USE_LTDL)
mpi->f ();
lt_dlclose (mpi->h);
mpi->h = QSE_NULL;
lt_dlexit ();
#endif
}
}
int qse_main (int argc, qse_achar_t* argv[])
{
int ret;
mpi_t mpi;
#if defined(_WIN32)
char locale[100];
@ -1162,15 +1241,11 @@ int qse_main (int argc, qse_achar_t* argv[])
qse_setdflcmgrbyid (QSE_CMGR_SLMB);
#endif
#if defined(USE_LTDL)
lt_dlinit ();
#endif
open_mpi (&mpi, argc, argv);
ret = qse_runmain (argc, argv, awk_main);
#if defined(USE_LTDL)
lt_dlexit ();
#endif
close_mpi (&mpi);
#if defined(_WIN32)
WSACleanup ();

2
qse/include/qse/awk/awk.h
View File

@ -1045,7 +1045,7 @@ enum qse_awk_trait_t
QSE_AWK_MODERN =
QSE_AWK_CLASSIC | QSE_AWK_EXTRAKWS | QSE_AWK_MAPTOVAR |
QSE_AWK_RWPIPE | QSE_AWK_REXBOUND | QSE_AWK_TOLERANT
QSE_AWK_RWPIPE | QSE_AWK_TOLERANT
};
/* ------------------------------------------------------------------------ */

24
qse/lib/awk/StdAwk.cpp
View File

@ -45,6 +45,7 @@
#else
# include <unistd.h>
# include <ltdl.h>
# define USE_LTDL
#endif
#ifndef QSE_HAVE_CONFIG_H
@ -134,8 +135,7 @@ int StdAwk::open ()
this->gbl_environ <= -1 ||
this->gbl_procinfo <= -1)
{
Awk::close ();
return -1;
goto oops;
}
if (addFunction (QSE_T("rand"), 0, 0, (FunctionHandler)&StdAwk::rand, 0) <= -1 ||
@ -145,10 +145,17 @@ int StdAwk::open ()
addFunction (QSE_T("setioattr"), 3, 3, (FunctionHandler)&StdAwk::setioattr, QSE_AWK_RIO) <= -1 ||
addFunction (QSE_T("getioattr"), 2, 2, (FunctionHandler)&StdAwk::getioattr, QSE_AWK_RIO) <= -1)
{
Awk::close ();
return -1;
goto oops;
}
#if defined(USE_LTDL)
/* lt_dlinit() can be called more than once and
* lt_dlexit() shuts down libltdl if it's called as many times as
* corresponding lt_dlinit(). so it's safe to call lt_dlinit()
* and lt_dlexit() at the library level. */
if (lt_dlinit() != 0) goto oops;
#endif
qse_ntime_t now;
this->seed = (qse_gettime(&now) <= -1)? 0u: (long_t)now;
@ -160,6 +167,10 @@ int StdAwk::open ()
this->cmgrtab_inited = false;
return 0;
oops:
Awk::close ();
return -1;
}
void StdAwk::close ()
@ -171,8 +182,13 @@ void StdAwk::close ()
this->cmgrtab_inited = false;
}
#endif
clearConsoleOutputs ();
Awk::close ();
#if defined(USE_LTDL)
lt_dlexit ();
#endif
}
StdAwk::Run* StdAwk::parse (Source& in, Source& out)

6
qse/lib/awk/run.c
View File

@ -827,14 +827,14 @@ void qse_awk_rtx_close (qse_awk_rtx_t* rtx)
qse_awk_rtx_ecb_t* ecb;
struct module_fini_ctx_t mfc;
for (ecb = rtx->ecb; ecb; ecb = ecb->next)
if (ecb->close) ecb->close (rtx);
mfc.limit = 0;
mfc.count = 0;
mfc.rtx = rtx;
qse_rbt_walk (rtx->awk->modtab, fini_module, &mfc);
for (ecb = rtx->ecb; ecb; ecb = ecb->next)
if (ecb->close) ecb->close (rtx);
/* NOTE:
* the close callbacks are called before data in rtx
* is destroyed. if the destruction count on any data

23
qse/lib/awk/std.c
View File

@ -49,6 +49,7 @@
#else
# include <unistd.h>
# include <ltdl.h>
# define USE_LTDL
#endif
#ifndef QSE_HAVE_CONFIG_H
@ -421,6 +422,9 @@ qse_awk_t* qse_awk_openstd (qse_size_t xtnsize)
static void fini_xtn (qse_awk_t* awk)
{
/* nothing to do */
#if defined(USE_LTDL)
lt_dlexit ();
#endif
}
static void clear_xtn (qse_awk_t* awk)
@ -447,6 +451,7 @@ qse_awk_t* qse_awk_openstdwithmmgr (qse_mmgr_t* mmgr, qse_size_t xtnsize)
prm.math.log10 = custom_awk_log10;
prm.math.exp = custom_awk_exp;
prm.math.sqrt = custom_awk_sqrt;
prm.modopen = custom_awk_modopen;
prm.modclose = custom_awk_modclose;
prm.modsym = custom_awk_modsym;
@ -460,17 +465,25 @@ qse_awk_t* qse_awk_openstdwithmmgr (qse_mmgr_t* mmgr, qse_size_t xtnsize)
/* add intrinsic global variables and functions */
if (add_globals(awk) <= -1 ||
add_functions (awk) <= -1)
{
qse_awk_close (awk);
return QSE_NULL;
}
add_functions (awk) <= -1) goto oops;
#if defined(USE_LTDL)
/* lt_dlinit() can be called more than once and
* lt_dlexit() shuts down libltdl if it's called as many times as
* corresponding lt_dlinit(). so it's safe to call lt_dlinit()
* and lt_dlexit() at the library level. */
if (lt_dlinit () != 0) goto oops;
#endif
xtn->ecb.close = fini_xtn;
xtn->ecb.clear = clear_xtn;
qse_awk_pushecb (awk, &xtn->ecb);
return awk;
oops:
if (awk) qse_awk_close (awk);
return QSE_NULL;
}
void* qse_awk_getxtnstd (qse_awk_t* awk)

170
qse/mod/awk/mpi.c
View File

@ -1,7 +1,38 @@
#include <qse/awk/std.h>
#include <qse/awk/awk.h>
#include <qse/cmn/str.h>
#include <qse/cmn/main.h>
#include <mpi.h>
#if defined(HAVE_MPI)
# include <mpi.h>
#else
# error this module needs mpi
#endif
static int fnc_size (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
{
qse_awk_val_t* retv;
int rank;
MPI_Comm_size (MPI_COMM_WORLD, &rank);
retv = qse_awk_rtx_makeintval (rtx, rank);
if (retv == QSE_NULL) return -1;
qse_awk_rtx_setretval (rtx, retv);
return 0;
}
static int fnc_rank (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
{
qse_awk_val_t* retv;
int rank;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
retv = qse_awk_rtx_makeintval (rtx, rank);
if (retv == QSE_NULL) return -1;
qse_awk_rtx_setretval (rtx, retv);
return 0;
}
static int fnc_hash (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
{
@ -27,10 +58,13 @@ static int fnc_assign (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
if (rx >= 0)
{
qse_awk_nrflt_t nrflt;
int size, rank;
MPI_Comm_size (MPI_COMM_WORLD, &size);
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
nrflt.limit = limit;
// nrflt.size = rxtn->size;
// nrflt.rank = rxtn->rank;
nrflt.size = size;
nrflt.rank = rank;
qse_awk_rtx_setnrflt (rtx, &nrflt);
}
@ -41,15 +75,12 @@ static int fnc_assign (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
return 0;
}
#if 0
static int fnc_reduce (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
{
qse_size_t nargs;
qse_awk_val_t* tmp, * a0, * a1;
qse_awk_val_t* retv;
qse_long_t opidx, lv;
qse_flt_t rv;
int n;
rxtn_t* rxtn;
static MPI_Op optab[] =
{
@ -61,55 +92,40 @@ static int fnc_reduce (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
MPI_LOR
};
rxtn = (rxtn_t*) qse_awk_rtx_getxtnstd (rtx);
nargs = qse_awk_rtx_getnargs (rtx);
QSE_ASSERT (nargs == 2);
a0 = qse_awk_rtx_getarg (rtx, 0);
a1 = qse_awk_rtx_getarg (rtx, 1);
if (qse_awk_rtx_valtolong (rtx, a1, &opidx) <= -1) return -1;
if (opidx < 0 || opidx >= QSE_COUNTOF(optab)) goto softfail;
if ((n = qse_awk_rtx_valtonum (rtx, a0, &lv, &rv)) <= -1) return -1;
if (qse_awk_rtx_valtolong (rtx, qse_awk_rtx_getarg (rtx, 1), &opidx) <= -1 ||
(opidx < 0 || opidx >= QSE_COUNTOF(optab)) ||
(n = qse_awk_rtx_valtonum (rtx, qse_awk_rtx_getarg (rtx, 0), &lv, &rv)) <= -1) goto softfail;
/* TODO: determine it to be MPI_LONG or MPI_INT, OR MPI_LONG_LONG_INT depending on the size of qse_long_t */
/* TODO: how to tell normal -1 from the soft failure??? */
if (n == 0)
{
qse_long_t lout;
if (MPI_Allreduce (&lv, &lout, 1, MPI_LONG_LONG_INT, optab[opidx], rxtn->comm) != MPI_SUCCESS) goto softfail;
tmp = qse_awk_rtx_makeintval (rtx, lout);
if (MPI_Allreduce (&lv, &lout, 1, MPI_LONG_LONG_INT, optab[opidx], MPI_COMM_WORLD) != MPI_SUCCESS) goto softfail;
retv = qse_awk_rtx_makeintval (rtx, lout);
}
else
{
qse_flt_t fout;
if (MPI_Allreduce (&rv, &fout, 1, MPI_LONG_DOUBLE, optab[opidx], rxtn->comm) != MPI_SUCCESS) goto softfail;
tmp = qse_awk_rtx_makefltval (rtx, fout);
if (MPI_Allreduce (&rv, &fout, 1, MPI_LONG_DOUBLE, optab[opidx], MPI_COMM_WORLD) != MPI_SUCCESS) goto softfail;
retv = qse_awk_rtx_makefltval (rtx, fout);
}
if (tmp == QSE_NULL) return -1;
qse_awk_rtx_setretval (rtx, tmp);
if (retv == QSE_NULL) return -1;
qse_awk_rtx_setretval (rtx, retv);
return 0;
softfail:
tmp = qse_awk_rtx_makeintval (rtx, (qse_long_t)-1);
if (tmp == QSE_NULL) return -1;
qse_awk_rtx_setretval (rtx, tmp);
/* return without setting the return value.
* this intrinsic function will return a nil value when it fails */
return 0;
}
#endif
static int fnc_barrier (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
{
int rx;
qse_awk_val_t* retv;
// rxtn_t* rxtn;
// rxtn = (rxtn_t*) qse_awk_rtx_getxtnstd (rtx);
// x = (MPI_Barrier (rxtn->comm) == MPI_SUCCESS)? 0: -1;
rx = (MPI_Barrier (MPI_COMM_WORLD) == MPI_SUCCESS)? 0: -1;
retv = qse_awk_rtx_makeintval (rtx, rx);
@ -119,43 +135,6 @@ static int fnc_barrier (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
return 0;
}
static int fnc_init (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
{
int rx;
qse_awk_val_t* retv;
/* I didn't manage to find a good way to change the
* default error handler to MPI_ERRORS_RETURN.
* so MPI_Init() will simply abort the program if it fails */
// if (MPI_Init (&argc, &argv) != MPI_SUCCESS) rx = -1;
// else
// {
MPI_Comm_set_errhandler (MPI_COMM_WORLD, MPI_ERRORS_RETURN);
rx = 0;
// }
retv = qse_awk_rtx_makeintval (rtx, rx);
if (retv == QSE_NULL) return -1;
qse_awk_rtx_setretval (rtx, retv);
return 0;
}
static int fnc_fini (qse_awk_rtx_t* rtx, const qse_awk_fnc_info_t* fi)
{
int rx;
qse_awk_val_t* retv;
MPI_Finalize ();
rx = 0;
retv = qse_awk_rtx_makeintval (rtx, rx);
if (retv == QSE_NULL) return -1;
qse_awk_rtx_setretval (rtx, retv);
return 0;
}
/* ------------------------------------------------------------------------- */
typedef struct fnctab_t fnctab_t;
@ -176,9 +155,20 @@ static fnctab_t fnctab[] =
{
{ QSE_T("assign"), { { 1, 1 }, fnc_assign } },
{ QSE_T("barrier"), { { 0, 0 }, fnc_barrier } },
{ QSE_T("fini"), { { 0, 0 }, fnc_fini } },
{ QSE_T("hash"), { { 1, 1 }, fnc_hash } },
{ QSE_T("init"), { { 0, 0 }, fnc_init } }
{ QSE_T("rank"), { { 0, 0 }, fnc_rank } },
{ QSE_T("reduce"), { { 2, 2 }, fnc_reduce } },
{ QSE_T("size"), { { 0, 0 }, fnc_size } }
};
static inttab_t inttab[] =
{
{ QSE_T("REDUCE_LAND"), { 4 } },
{ QSE_T("REDUCE_LOR"), { 5 } },
{ QSE_T("REDUCE_MAX"), { 1 } },
{ QSE_T("REDUCE_MIN"), { 0 } },
{ QSE_T("REDUCE_PROD"), { 3 } },
{ QSE_T("REDUCE_SUM"), { 2 } }
};
static int query (qse_awk_mod_t* mod, qse_awk_t* awk, const qse_char_t* name, qse_awk_mod_sym_t* sym)
@ -197,7 +187,6 @@ static int query (qse_awk_mod_t* mod, qse_awk_t* awk, const qse_char_t* name, qs
}
}
#if 0
for (i = 0; i < QSE_COUNTOF(inttab); i++)
{
if (qse_strcmp (inttab[i].name, name) == 0)
@ -207,7 +196,6 @@ static int query (qse_awk_mod_t* mod, qse_awk_t* awk, const qse_char_t* name, qs
return 0;
}
}
#endif
ea.ptr = name;
ea.len = qse_strlen(name);
@ -217,12 +205,13 @@ static int query (qse_awk_mod_t* mod, qse_awk_t* awk, const qse_char_t* name, qs
/* TODO: proper resource management */
int init (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
static int init (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
{
/* TODO: anything */
return 0;
}
void fini (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
static void fini (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
{
/* TODO: anything */
}
@ -246,3 +235,30 @@ int load (qse_awk_mod_t* mod, qse_awk_t* awk)
return 0;
}
/* The MPI module is special in that it exports 2 extra symbols -
* mpi_init and mpi_fini. These two symbols are intended to be called
* dynamically using dlopen() or something similar when an application
* intending to use mpi::xxx starts up. This way, the application doesn't
* have to be linked to any MPI libraries while this module is linked to
* an MPI library. If this module doesn't exist, it means MPI is not availble
* and the module wasn't built. So you can't access mpi::xxx symbols either
*/
int mpi_init (int argc, qse_achar_t* argv[])
{
int rx;
if (MPI_Init (&argc, &argv) != MPI_SUCCESS) rx = -1;
else
{
MPI_Comm_set_errhandler (MPI_COMM_WORLD, MPI_ERRORS_RETURN);
rx = 0;
}
return rx;
}
void mpi_fini (void)
{
MPI_Finalize ();
}

6
qse/mod/awk/sys.c
View File

@ -1,4 +1,4 @@
#include <qse/awk/std.h>
#include <qse/awk/awk.h>
#include <qse/cmn/str.h>
#if defined(_WIN32)
@ -298,12 +298,12 @@ static int query (qse_awk_mod_t* mod, qse_awk_t* awk, const qse_char_t* name, qs
/* TODO: proper resource management */
int init (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
static int init (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
{
return 0;
}
void fini (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
static void fini (qse_awk_mod_t* mod, qse_awk_rtx_t* rtx)
{
/* TODO:
for (each pid for rtx) kill (pid, SIGKILL);

2
qse/mod/awk/uci.c
View File

@ -1,4 +1,4 @@
#include <qse/awk/std.h>
#include <qse/awk/awk.h>
#include <qse/cmn/str.h>
#include <qse/cmn/rbt.h>
#include <qse/cmn/mbwc.h>

12
qse/samples/awk/awk08.cpp
View File

@ -36,8 +36,6 @@
# include <unistd.h>
# include <signal.h>
# include <errno.h>
# include <ltdl.h>
# define USE_LTDL
#endif
/* these three definitions for doxygen cross-reference */
@ -58,6 +56,8 @@ public:
idLastSleep = addGlobal (QSE_T("LAST_SLEEP"));
if (idLastSleep <= -1) goto oops;
/* this is for demonstration only.
* you can use sys::sleep() instead */
if (addFunction (QSE_T("sleep"), 1, 1,
(FunctionHandler)&MyAwk::sleep) <= -1) goto oops;
@ -470,16 +470,8 @@ int qse_main (int argc, qse_achar_t* argv[])
qse_setdflcmgrbyid (QSE_CMGR_SLMB);
#endif
#if defined(USE_LTDL)
lt_dlinit ();
#endif
ret = qse_runmain (argc, argv, awk_main);
#if defined(USE_LTDL)
lt_dlexit ();
#endif
#if defined(_WIN32)
WSACleanup ();
#endif