MACSio  0.9
Multi-purpose, Application-Centric, Scalable I/O Proxy App
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
macsio_mif.c
Go to the documentation of this file.
1 /*
2 Copyright (c) 2015, Lawrence Livermore National Security, LLC.
3 Produced at the Lawrence Livermore National Laboratory.
4 Written by Mark C. Miller
5 
6 LLNL-CODE-676051. All rights reserved.
7 
8 This file is part of MACSio
9 
10 Please also read the LICENSE file at the top of the source code directory or
11 folder hierarchy.
12 
13 This program is free software; you can redistribute it and/or modify it under
14 the terms of the GNU General Public License (as published by the Free Software
15 Foundation) version 2, dated June 1991.
16 
17 This program is distributed in the hope that it will be useful, but WITHOUT
18 ANY WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS
19 FOR A PARTICULAR PURPOSE. See the terms and conditions of the GNU General
20 Public License for more details.
21 
22 You should have received a copy of the GNU General Public License along with
23 this program; if not, write to the Free Software Foundation, Inc., 59 Temple
24 Place, Suite 330, Boston, MA 02111-1307 USA
25 */
26 
27 #include <stdlib.h>
28 
29 #ifdef HAVE_SCR
30 #ifdef __cplusplus
31 extern "C" {
32 #endif
33 #include <scr.h>
34 #ifdef __cplusplus
35 }
36 #endif
37 #endif
38 
39 #include <macsio_mif.h>
40 
41 #define MACSIO_MIF_BATON_OK 0
42 #define MACSIO_MIF_BATON_ERR 1
43 #define MACSIO_MIF_MIFMAX -1
44 #define MACSIO_MIF_MIFAUTO -2
45 
51 typedef struct _MACSIO_MIF_baton_t
52 {
54 #ifdef HAVE_MPI
55  MPI_Comm mpiComm;
56 #endif
57  int commSize;
58  int rankInComm;
59  int numGroups;
61  int groupSize;
62  int groupRank;
63  int commSplit;
67  mutable int mifErr;
68  mutable int mpiErr;
69  int mpiTag;
73  void *clientData;
75 
100 #warning ADD A THROTTLE OPTION HERE FOR TOT FILES VS CONCURRENT FILES
101 #warning FOR AUTO MODE, MUST HAVE A CALL TO QUERY FILE COUNT
103  int numFiles,
109  MACSIO_MIF_ioFlags_t ioFlags,
110 #ifdef HAVE_MPI
111  MPI_Comm mpiComm,
113 #else
114  int mpiComm,
115 #endif
116  int mpiTag,
118  MACSIO_MIF_CreateCB createCb,
119  MACSIO_MIF_OpenCB openCb,
120  MACSIO_MIF_CloseCB closeCb,
121  void *clientData
122 )
123 {
124  int numGroups = numFiles;
125  int commSize, rankInComm;
126  int groupSize, numGroupsWithExtraProc, commSplit,
127  groupRank, rankInGroup, procBeforeMe, procAfterMe;
128  MACSIO_MIF_baton_t *ret = 0;
129 
130  procBeforeMe = -1;
131  procAfterMe = -1;
132 
133  MPI_Comm_size(mpiComm, &commSize);
134  MPI_Comm_rank(mpiComm, &rankInComm);
135 
136  groupSize = commSize / numGroups;
137  numGroupsWithExtraProc = commSize % numGroups;
138  commSplit = numGroupsWithExtraProc * (groupSize + 1);
139 
140  if (rankInComm < commSplit)
141  {
142  groupRank = rankInComm / (groupSize + 1);
143  rankInGroup = rankInComm % (groupSize + 1);
144  if (rankInGroup < groupSize)
145  procAfterMe = rankInComm + 1;
146  }
147  else
148  {
149  groupRank = numGroupsWithExtraProc + (rankInComm - commSplit) / groupSize;
150  rankInGroup = (rankInComm - commSplit) % groupSize;
151  if (rankInGroup < groupSize - 1)
152  procAfterMe = rankInComm + 1;
153  }
154  if (rankInGroup > 0)
155  procBeforeMe = rankInComm - 1;
156 
157  if (createCb == 0 || openCb == 0 || closeCb == 0)
158  return 0;
159 
160  ret = (MACSIO_MIF_baton_t *) malloc(sizeof(MACSIO_MIF_baton_t));
161  ret->ioFlags = ioFlags;
162  ret->commSize = commSize;
163  ret->rankInComm = rankInComm;
164  ret->numGroups = numGroups;
165  ret->groupSize = groupSize;
166  ret->numGroupsWithExtraProc = numGroupsWithExtraProc;
167  ret->commSplit = commSplit;
168  ret->groupRank = groupRank;
169  ret->rankInGroup = rankInGroup;
170  ret->procBeforeMe = procBeforeMe;
171  ret->procAfterMe = procAfterMe;
173 #ifdef HAVE_MPI
174  ret->mpiErr = MPI_SUCCESS;
175 #else
176  ret->mpiErr = 0;
177 #endif
178  ret->mpiTag = mpiTag;
179  ret->mpiComm = mpiComm;
180  ret->createCb = createCb;
181  ret->openCb = openCb;
182  ret->closeCb = closeCb;
183  ret->clientData = clientData;
184 
185  return ret;
186 }
187 
192  MACSIO_MIF_baton_t *bat
193 )
194 {
195  free(bat);
196 }
197 
211  MACSIO_MIF_baton_t *Bat,
212  char const *fname,
213  char const *nsname
214 )
215 {
216  if (Bat->procBeforeMe != -1)
217  {
218  MPI_Status mpi_stat;
219  int baton;
220  int mpi_err = MPI_Recv(&baton, 1, MPI_INT, Bat->procBeforeMe,
221  Bat->mpiTag, Bat->mpiComm, &mpi_stat);
222  if (mpi_err == MPI_SUCCESS && baton != MACSIO_MIF_BATON_ERR)
223  {
224 #ifdef HAVE_SCR
225  char scr_filename[SCR_MAX_FILENAME];
226  if (Bat->ioFlags.use_scr)
227  {
228  if (SCR_Route_file(fname, scr_filename) == SCR_SUCCESS)
229  return Bat->openCb(scr_filename, nsname, Bat->ioFlags, Bat->clientData);
230  else
231  return Bat->openCb(fname, nsname, Bat->ioFlags, Bat->clientData);
232  }
233  else
234 #endif
235  return Bat->openCb(fname, nsname, Bat->ioFlags, Bat->clientData);
236  }
237  else
238  {
240  Bat->mpiErr = mpi_err;
241  return 0;
242  }
243  }
244  else
245  {
246  if (Bat->ioFlags.do_wr)
247  {
248 #ifdef HAVE_SCR
249  char scr_filename[SCR_MAX_FILENAME];
250  if (Bat->ioFlags.use_scr)
251  {
252  SCR_Route_file(fname, scr_filename);
253  return Bat->createCb(scr_filename, nsname, Bat->clientData);
254  }
255  else
256 #endif
257  return Bat->createCb(fname, nsname, Bat->clientData);
258  }
259  else
260  {
261 #ifdef HAVE_SCR
262  char scr_filename[SCR_MAX_FILENAME];
263  if (Bat->ioFlags.use_scr)
264  {
265  if (SCR_Route_file(fname, scr_filename) == SCR_SUCCESS)
266  return Bat->openCb(scr_filename, nsname, Bat->ioFlags, Bat->clientData);
267  else
268  return Bat->openCb(fname, nsname, Bat->ioFlags, Bat->clientData);
269  }
270  else
271 #endif
272  return Bat->openCb(fname, nsname, Bat->ioFlags, Bat->clientData);
273  }
274  }
275 }
276 
285  MACSIO_MIF_baton_t const *Bat,
286  void *file
287 )
288 {
289  Bat->closeCb(file, Bat->clientData);
290  if (Bat->procAfterMe != -1)
291  {
292  int baton = Bat->mifErr;
293  int mpi_err = MPI_Ssend(&baton, 1, MPI_INT, Bat->procAfterMe,
294  Bat->mpiTag, Bat->mpiComm);
295  if (mpi_err != MPI_SUCCESS)
296  {
298  Bat->mpiErr = mpi_err;
299  }
300  }
301 }
302 
312  MACSIO_MIF_baton_t const *Bat,
313  int rankInComm
314 )
315 {
316  int retval;
317 
318  if (rankInComm < Bat->commSplit)
319  {
320  retval = rankInComm / (Bat->groupSize + 1);
321  }
322  else
323  {
324  retval = Bat->numGroupsWithExtraProc +
325  (rankInComm - Bat->commSplit) / Bat->groupSize;
326  }
327 
328  return retval;
329 }
330 
340  MACSIO_MIF_baton_t const *Bat,
341  int rankInComm
342 )
343 {
344  int retval;
345 
346  if (rankInComm < Bat->commSplit)
347  {
348  retval = rankInComm % (Bat->groupSize + 1);
349  }
350  else
351  {
352  retval = (rankInComm - Bat->commSplit) % Bat->groupSize;
353  }
354 
355  return retval;
356 }
357 
#define MACSIO_MIF_BATON_OK
Definition: macsio_mif.c:41
unsigned int use_scr
Definition: macsio_mif.h:156
int MACSIO_MIF_RankOfGroup(MACSIO_MIF_baton_t const *Bat, int rankInComm)
Rank of the group in which a given (global) rank exists.
Definition: macsio_mif.c:311
unsigned int do_wr
Definition: macsio_mif.h:155
void * MACSIO_MIF_WaitForBaton(MACSIO_MIF_baton_t *Bat, char const *fname, char const *nsname)
Wait for exclusive access to the group's file.
Definition: macsio_mif.c:210
void *(* MACSIO_MIF_CreateCB)(const char *fname, const char *nsname, void *udata)
Definition: macsio_mif.h:160
MACSIO_MIF_CloseCB closeCb
Definition: macsio_mif.c:72
void MACSIO_MIF_Finish(MACSIO_MIF_baton_t *bat)
End a MACSIO_MIF I/O operation and free resources.
Definition: macsio_mif.c:191
MACSIO_MIF_ioFlags_t ioFlags
Definition: macsio_mif.c:53
void(* MACSIO_MIF_CloseCB)(void *file, void *udata)
Definition: macsio_mif.h:164
MACSIO_MIF_CreateCB createCb
Definition: macsio_mif.c:70
void *(* MACSIO_MIF_OpenCB)(const char *fname, const char *nsname, MACSIO_MIF_ioFlags_t ioFlags, void *udata)
Definition: macsio_mif.h:161
MACSIO_MIF_baton_t * MACSIO_MIF_Init(int numFiles, MACSIO_MIF_ioFlags_t ioFlags, MPI_Comm mpiComm, int mpiTag, MACSIO_MIF_CreateCB createCb, MACSIO_MIF_OpenCB openCb, MACSIO_MIF_CloseCB closeCb, void *clientData)
Initialize MACSIO_MIF for a MIF I/O operation.
Definition: macsio_mif.c:102
int MACSIO_MIF_RankInGroup(MACSIO_MIF_baton_t const *Bat, int rankInComm)
Rank within a group of a given (global) rank.
Definition: macsio_mif.c:339
#define MACSIO_MIF_BATON_ERR
Definition: macsio_mif.c:42
MACSIO_MIF_OpenCB openCb
Definition: macsio_mif.c:71
struct _MACSIO_MIF_baton_t MACSIO_MIF_baton_t
void MACSIO_MIF_HandOffBaton(MACSIO_MIF_baton_t const *Bat, void *file)
Release exclusive access to the group's file.
Definition: macsio_mif.c:284