MACSio  0.9
Multi-purpose, Application-Centric, Scalable I/O Proxy App
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
macsio_miftmpl.c
Go to the documentation of this file.
1 /*
2 Copyright (c) 2015, Lawrence Livermore National Security, LLC.
3 Produced at the Lawrence Livermore National Laboratory.
4 Written by Mark C. Miller
5 
6 LLNL-CODE-676051. All rights reserved.
7 
8 This file is part of MACSio
9 
10 Please also read the LICENSE file at the top of the source code directory or
11 folder hierarchy.
12 
13 This program is free software; you can redistribute it and/or modify it under
14 the terms of the GNU General Public License (as published by the Free Software
15 Foundation) version 2, dated June 1991.
16 
17 This program is distributed in the hope that it will be useful, but WITHOUT
18 ANY WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS
19 FOR A PARTICULAR PURPOSE. See the terms and conditions of the GNU General
20 Public License for more details.
21 
22 You should have received a copy of the GNU General Public License along with
23 this program; if not, write to the Free Software Foundation, Inc., 59 Temple
24 Place, Suite 330, Boston, MA 02111-1307 USA
25 */
26 
27 #include <macsio_clargs.h>
28 #include <macsio_iface.h>
29 #include <macsio_log.h>
30 #include <macsio_main.h>
31 #include <macsio_mif.h>
32 #include <macsio_utils.h>
33 
34 #include <stdio.h>
35 
36 #ifdef HAVE_MPI
37 #include <mpi.h>
38 #endif
39 
45 #warning ADD PWRITE OPTION
46 #warning ADD MPI_FILE_WRITE OPTION
47 #warning ADD AGGREGATION OPTION
48 
177 static char const *iface_name = "miftmpl";
178 static char const *iface_ext = "json";
179 static int json_as_html = 0;
180 static int my_opt_one;
182 static int my_opt_two;
183 static char *my_opt_three_string;
184 static float my_opt_three_float;
196 static int process_args(
197  int argi,
198  int argc,
199  char *argv[]
200 )
201 {
202  /* Can use MACSIO_CLARGS_TOJSON here instead in which case pass the pointer to
203  a json_object* as first arg and eliminate all the pointers to specific
204  variables. The args will be returned as a json-c object. */
206 
207  MACSIO_CLARGS_ProcessCmdline(0, argFlags, argi, argc, argv,
208  "--json_as_html", "",
209  "Write files as HTML instead of raw ascii [false]",
210  &json_as_html,
211  "--my_opt_one", "",
212  "Help message for my_opt_one which has no arguments. If present, local\n"
213  "var my_opt_one will be assigned a value of 1 and a value of zero otherwise.",
214  &my_opt_one,
215  "--my_opt_two %d", MACSIO_CLARGS_NODEFAULT,
216  "Help message for my_opt_two which has a single integer argument",
217  &my_opt_two,
218  "--my_opt_three %s %f", MACSIO_CLARGS_NODEFAULT,
219  "Help message for my_opt_three which has a string argument and a float argument",
222 
223  return 0;
224 }
225 
233 static void *CreateMyFile(
234  const char *fname,
235  const char *nsname,
236  void *userData
237 )
238 {
239  FILE *file = fopen(fname, "w");
240  return (void *) file;
241 }
242 
250 static void *OpenMyFile(
251  const char *fname,
252  const char *nsname,
253  MACSIO_MIF_ioFlags_t ioFlags,
254  void *userData
255 )
256 {
257  FILE *file = fopen(fname, "a+");
258  return (void *) file;
259 }
260 
266 static void CloseMyFile(
267  void *file,
268  void *userData
269 )
270 {
271  fclose((FILE*) file);
272 }
273 
288  FILE *myFile,
289  char const *fileName,
290  json_object *part_obj
291 )
292 {
293  json_object *part_info = json_object_new_object();
294 
295 #warning SOMEHOW SHOULD INCLUDE OFFSETS TO EACH VARIABLE
296  /* Write the json mesh part object as an ascii string */
297  fprintf(myFile, "%s\n", json_object_to_json_string_ext(part_obj, JSON_C_TO_STRING_PRETTY));
298  json_object_free_printbuf(part_obj);
299 
300  /* Form the return 'value' holding the information on where to find this part */
301  json_object_object_add(part_info, "partid",
302 #warning CHANGE NAME OF KEY IN JSON TO PartID
303  json_object_new_int(json_object_path_get_int(part_obj, "Mesh/ChunkID")));
304  json_object_object_add(part_info, "file",
305  json_object_new_string(fileName));
306  json_object_object_add(part_info, "offset",
307  json_object_new_double((double) ftello(myFile)));
308 
309  return part_info;
310 }
311 
324 static void main_dump(
325  int argi,
326  int argc,
327  char **argv,
328  json_object *main_obj,
329  int dumpn,
331  double dumpt
332 )
333 {
334  int i, rank, numFiles;
335  char fileName[256];
336  FILE *myFile;
337  MACSIO_MIF_ioFlags_t ioFlags = {MACSIO_MIF_WRITE, JsonGetInt(main_obj, "clargs/exercise_scr")&0x1};
338  MACSIO_MIF_baton_t *bat;
339  json_object *parts;
340  json_object *part_infos = json_object_new_array();
341 
342  /* process cl args */
343  process_args(argi, argc, argv);
344 
345  /* ensure we're in MIF mode and determine the file count */
346 #warning SIMPLIFY THIS LOGIC USING NEW JSON INTERFACE
347  json_object *parfmode_obj = json_object_path_get_array(main_obj, "clargs/parallel_file_mode");
348  if (parfmode_obj)
349  {
350  json_object *modestr = json_object_array_get_idx(parfmode_obj, 0);
351  json_object *filecnt = json_object_array_get_idx(parfmode_obj, 1);
352  if (!strcmp(json_object_get_string(modestr), "SIF"))
353  {
354  MACSIO_LOG_MSG(Die, ("miftmpl plugin cannot currently handle SIF mode"));
355  }
356  else
357  {
358  numFiles = json_object_get_int(filecnt);
359  }
360  }
361  else
362  {
363  char const * modestr = json_object_path_get_string(main_obj, "clargs/parallel_file_mode");
364  if (!strcmp(modestr, "SIF"))
365  {
366  MACSIO_LOG_MSG(Die, ("miftmpl plugin cannot currently handle SIF mode"));
367  }
368  else if (!strcmp(modestr, "MIFMAX"))
369  numFiles = json_object_path_get_int(main_obj, "parallel/mpi_size");
370  else if (!strcmp(modestr, "MIFAUTO"))
371  {
372  /* Call MACSio utility to determine optimal file count */
373  }
374  }
375 
376  bat = MACSIO_MIF_Init(numFiles, ioFlags, MACSIO_MAIN_Comm, 3,
378 
379  rank = json_object_path_get_int(main_obj, "parallel/mpi_rank");
380 
381  /* Construct name for the silo file */
382  sprintf(fileName, "%s_json_%05d_%03d.%s",
383  json_object_path_get_string(main_obj, "clargs/filebase"),
384  MACSIO_MIF_RankOfGroup(bat, rank),
385  dumpn,
386  json_object_path_get_string(main_obj, "clargs/fileext"));
387 
388  myFile = (FILE *) MACSIO_MIF_WaitForBaton(bat, fileName, 0);
389 
390  parts = json_object_path_get_array(main_obj, "problem/parts");
391  for (int i = 0; i < json_object_array_length(parts); i++)
392  {
393  json_object *this_part = json_object_array_get_idx(parts, i);
394  json_object_array_add(part_infos, write_mesh_part(myFile, fileName, this_part));
395  }
396 
397  /* Hand off the baton to the next processor. This winds up closing
398  * the file so that the next processor that opens it can be assured
399  * of getting a consistent and up to date view of the file's contents. */
400  MACSIO_MIF_HandOffBaton(bat, myFile);
401 
402  /* We're done using MACSIO_MIF for these files, so finish it off */
403  MACSIO_MIF_Finish(bat);
404 
405  /* Use MACSIO_MIF a second time to manage writing of the master/root
406  file contents. This winds up being serial I/O but also means we
407  never collect all info on all parts to any single processor. */
408 #warning THERE IS A BETTER WAY TO DO USING LOOP OF NON-BLOCKING RECIEVES
409  bat = MACSIO_MIF_Init(1, ioFlags, MACSIO_MAIN_Comm, 5,
411 
412  /* Construct name for the silo file */
413  sprintf(fileName, "%s_json_root_%03d.%s",
414  json_object_path_get_string(main_obj, "clargs/filebase"),
415  dumpn,
416  json_object_path_get_string(main_obj, "clargs/fileext"));
417 
418  /* Wait for MACSIO_MIF to give this processor exclusive access */
419  myFile = (FILE *) MACSIO_MIF_WaitForBaton(bat, fileName, 0);
420 
421 #warning FIX THE STRING THAT WE PRODUCE HERE SO ITS A SINGLE JSON ARRAY OBJECT
422  /* This processor's work on the file is just to write its part_infos */
423  fprintf(myFile, "%s\n", json_object_to_json_string_ext(part_infos, JSON_C_TO_STRING_PRETTY));
424 
425  MACSIO_MIF_HandOffBaton(bat, myFile);
426 
427  MACSIO_MIF_Finish(bat);
428 
429  /* decriment ref-count (and free) part_infos */
430  json_object_put(part_infos);
431 }
432 
443 {
444  MACSIO_IFACE_Handle_t iface;
445 
446  if (strlen(iface_name) >= MACSIO_IFACE_MAX_NAME)
447  MACSIO_LOG_MSG(Die, ("Interface name \"%s\" too long", iface_name));
448 
449  /* Populate information about this plugin */
450  strcpy(iface.name, iface_name);
451  strcpy(iface.ext, iface_ext);
452  iface.dumpFunc = main_dump;
454 
455  /* Register this plugin */
456  if (!MACSIO_IFACE_Register(&iface))
457  MACSIO_LOG_MSG(Die, ("Failed to register interface \"%s\"", iface_name));
458 
459  return 0;
460 }
461 
473 static int const dummy = register_this_interface();
474 
int MACSIO_MIF_RankOfGroup(MACSIO_MIF_baton_t const *Bat, int rankInComm)
Rank of the group in which a given (global) rank exists.
Definition: macsio_mif.c:311
struct json_object * json_object_new_string(const char *s)
Definition: json_object.c:727
void * MACSIO_MIF_WaitForBaton(MACSIO_MIF_baton_t *Bat, char const *fname, char const *nsname)
Wait for exclusive access to the group's file.
Definition: macsio_mif.c:210
#define MACSIO_CLARGS_WARN
Definition: macsio_clargs.h:35
struct json_object * json_object_new_object(void)
Definition: json_object.c:369
#define MACSIO_MIF_WRITE
Definition: macsio_mif.h:149
int MACSIO_CLARGS_ProcessCmdline(void **retobj, MACSIO_CLARGS_ArgvFlags_t flags, int argi, int argc, char **argv,...)
static void * CreateMyFile(const char *fname, const char *nsname, void *userData)
CreateFile MIF Callback.
int MACSIO_IFACE_Register(MACSIO_IFACE_Handle_t const *iface)
Definition: macsio_iface.c:35
void MACSIO_MIF_Finish(MACSIO_MIF_baton_t *bat)
End a MACSIO_MIF I/O operation and free resources.
Definition: macsio_mif.c:191
static int process_args(int argi, int argc, char *argv[])
Process command-line arguments specific to this plugin.
ProcessArgsFunc processArgsFunc
Definition: macsio_iface.h:59
struct json_object * json_object_new_int(int32_t i)
Definition: json_object.c:495
#define MACSIO_IFACE_MAX_NAME
Definition: macsio_iface.h:33
int json_object_array_add(struct json_object *obj, struct json_object *val)
Definition: json_object.c:859
static char const * iface_name
int json_object_array_length(struct json_object *obj)
Definition: json_object.c:854
static float my_opt_three_float
#define JsonGetInt(OBJ,...)
Get the int value at specified path.
Definition: json_object.h:747
static int json_as_html
struct json_object * json_object_new_array(void)
Definition: json_object.c:828
static void * OpenMyFile(const char *fname, const char *nsname, MACSIO_MIF_ioFlags_t ioFlags, void *userData)
OpenFile MIF Callback.
#define MACSIO_CLARGS_END_OF_ARGS
Definition: macsio_clargs.h:53
#define MACSIO_CLARGS_NODEFAULT
Definition: macsio_clargs.h:54
struct json_object * json_object_path_get_array(struct json_object *src, char const *key_path)
Get the array object at specified path.
Definition: json_object.c:2215
const char * json_object_get_string(struct json_object *obj)
Definition: json_object.c:751
struct json_object * json_object_new_double(double d)
Definition: json_object.c:622
char ext[MACSIO_IFACE_MAX_NAME]
Definition: macsio_iface.h:55
char const * json_object_path_get_string(struct json_object *src, char const *key_path)
Get a string value for the object at specified path.
Definition: json_object.c:2198
#define MACSIO_CLARGS_TOMEM
Definition: macsio_clargs.h:39
struct json_object * json_object_array_get_idx(struct json_object *obj, int idx)
Definition: json_object.c:870
#define JSON_C_TO_STRING_PRETTY
Definition: json_object.h:55
MACSIO_MIF_baton_t * MACSIO_MIF_Init(int numFiles, MACSIO_MIF_ioFlags_t ioFlags, MPI_Comm mpiComm, int mpiTag, MACSIO_MIF_CreateCB createCb, MACSIO_MIF_OpenCB openCb, MACSIO_MIF_CloseCB closeCb, void *clientData)
Initialize MACSIO_MIF for a MIF I/O operation.
Definition: macsio_mif.c:102
char name[MACSIO_IFACE_MAX_NAME]
Definition: macsio_iface.h:54
int json_object_put(struct json_object *obj)
Definition: json_object.c:155
static int my_opt_one
int32_t json_object_get_int(struct json_object *obj)
Definition: json_object.c:504
const char * json_object_to_json_string_ext(struct json_object *obj, int flags)
Definition: json_object.c:277
int32_t json_object_path_get_int(struct json_object *src, char const *key_path)
Get integer value for object at specified path.
Definition: json_object.c:2103
static char const * iface_ext
static int const dummy
Dummy initializer to trigger register_this_interface by the loader.
MPI_Comm MACSIO_MAIN_Comm
Definition: macsio_main.c:279
static json_object * write_mesh_part(FILE *myFile, char const *fileName, json_object *part_obj)
Write a single mesh part to a MIF file.
static int register_this_interface()
Method to register this plugin with MACSio main.
static void main_dump(int argi, int argc, char **argv, json_object *main_obj, int dumpn, double dumpt)
Main MIF dump implementation for this plugin.
void MACSIO_MIF_HandOffBaton(MACSIO_MIF_baton_t const *Bat, void *file)
Release exclusive access to the group's file.
Definition: macsio_mif.c:284
void json_object_free_printbuf(struct json_object *obj)
Definition: json_object.c:2288
static void CloseMyFile(void *file, void *userData)
CloseFile MIF Callback.
#define MACSIO_LOG_MSG(SEV, MSG)
Convenience macro for logging a message to the main log.
Definition: macsio_log.h:133
void json_object_object_add(struct json_object *obj, const char *key, struct json_object *val)
Definition: json_object.c:391
static char * my_opt_three_string
static int my_opt_two