Issue with each

1

Dear,

I am trying to load some mapping from OCL into collection but I cannot manage to get it working

here my job

// Check out the Job Writing Guide for help getting started:
// https://docs.openfn.org/documentation/jobs/job-writing-guide
fn(state => {
  state.configuration.baseUrl = state.configuration.hostUrl;
  console.log(state.configuration.baseUrl);
  return state;
});

get(
  "orgs/SwissTPH/sources/ALM-DHIS2/HEAD/mappings",
  {
    limit: 200
  },
  (state) => {
    state.recievedMappingsList = state.data
    state.mappingsList = state.mappingsList || [];
    return state
  }
);
fn((state) => { 
  // Remove existing collection entries
  collections.remove('global-almanach-dhis2', { createdAfter: '202401' });
   console.log(state.recievedMappingsList.length)
  // load new entry after getting the datatype from the dhis2 concept
  each(`$.data[*]`, (state) => {
    console.log(state.data)
    mapping = JSON.parse(state.data)
    get(mapping.from_concept_url.slice(1), (state) => {
          const type = mapping.datatype || 'unknown'; // Fallback if datatype is undefined
          const name = mapping.to_concept_code.replace(/[. -]/g, "_")
          const item = {
            map_type: mapping.map_type,
            dhis2_uid: mapping.from_concept_code,
            name: name,
            datatype: type,
          }
          console.log(name)
          // Append to mappingsList
          state.mappingList.push(item)
          collections.set('global-almanach-dhis2',name,item);
          return state;
        });
        return state;
  }
  )
      return  state
})

it seems like the inner function of the each does nothing

I tried many version but only one where I managed to get some log is this one but it shows only one log entry in a showing the 22 records in a json line mode

  each(`$.data[*]`,
       console.log(state.data)
  )

even this one gives nothing

  each(`$.data[*]`, (state) =>
       console.log(state.data)
  )
2

Weirder

JSON.parse(state.data)

works outside the each but not within (trigger a js error)

3

Hello @delcroip ,

Looking at your implementation, there are a few things to note:

  • each() is an operational function and cannot be used inside an fn() block.
  • collections() can be used as standalone functions and not inside an fn() block.
  • each() takes two arguments, the data and the function call as this example demonstrates
each(
  $.data,
  // Inside the callback operation, `$.data` is scoped to the item under iteration
  insert("patient", {
    patient_name: $.data.properties.case_name,
    patient_id: $.data.case_id,
  })
);
  • We can parse this section mapping = JSON.parse(state.data) inside an fn() block and then use it in the each() .

So something like this would be the change needed:

// Check out the Job Writing Guide for help getting started:
// https://docs.openfn.org/documentation/jobs/job-writing-guide
fn(state => {
  state.configuration.baseUrl = state.configuration.hostUrl;
  console.log(state.configuration.baseUrl);
  return state;
});

get(
  'orgs/SwissTPH/sources/ALM-DHIS2/HEAD/mappings',
  {
    limit: 200,
  },
  state => {
    state.recievedMappingsList = state.data;
    state.mappingsList = state.mappingsList || [];
    return state;
  }
);

// Remove existing collection entries
collections.remove('global-almanach-dhis2', { createdAfter: '202401' });

fn(state => {
  console.log(state.recievedMappingsList.length);
  state.mapping = JSON.parse(state.data);

  return state;
});

// load new entry after getting the datatype from the dhis2 concept
each(
  `$.mapping[*]`,
  get($.data.from_concept_url.slice(1), state => {
    return state;
  })
);

Do you mind trying the fixes out? I would be happy to assist further incase something is not clear

4

thank you, moving the each on the root helped but I am now failing to get the datatype

any idea on how I could structure the code ?

// Check out the Job Writing Guide for help getting started:
// https://docs.openfn.org/documentation/jobs/job-writing-guide
fn(state => {
  state.configuration.baseUrl = state.configuration.hostUrl;
  console.log(state.configuration.baseUrl);
  return state;
});

get(
  "orgs/SwissTPH/sources/ALM-DHIS2/HEAD/mappings",
  {
    limit: 200
  },
  (state) => {
    state.recievedMappingsList = state.data
    state.mappingsList = state.mappingsList || [];
    return state
  }
);


  // Remove existing collection entries
  //collections.remove('global-almanach-dhis2', { createdAfter: '202401' });
  // load new entry after getting the datatype from the dhis2 concept
  each(`$.recievedMappingsList[*]`,  (state) => {
    state.curMapping = state.data
    const type = get(state.data.from_concept_url.slice(1), (state) => {
          console.log(state.data);
          return state.data.datatype || 'unknown';
        });
     
    const name = state.curMapping.to_concept_code.replace(/[. -]/g, "_")
      const item = {
        map_type: state.curMapping.map_type,
        dhis2_uid: state.curMapping.from_concept_code,
        name: name,
        datatype: type,
      }
      console.log(item)
      // Append to mappingsList
      state.mappingsList.push(item)
      collections.set('global-almanach-dhis2',name,item);
    return state;
  }
  )
5

Retarding the example none was matching my usecase where the each callback was with the format (state) => {}

So I found out that the item in the loop is in state.data (kind of expected) even if the loop is on $.mapping[*]

6

Based on your example, I think you do not need to do this:

each(`$.recievedMappingsList[*]`, (state)=>{
get()})

Instead, try this:

each(`$.recievedMappingsList[*]`, get(state.data.from_concept_url.slice(1)))

When you use each(), every single item will be mapped into state.data. With this in mind, we therefore do not need to nest the get() inside another operation.
To achieve the desired output using after calling get() within each(), you can use promise chaining after get() where: get().then(state => {}).
Please try this and we can iterate again

1 Like
7

Using the promise worked BUT I had to use a trick to save the curMapping

each(`$.recievedMappingsList[*]`, get((state) => {   state.curMapping = state.data; return state.data.from_concept_url.slice(1)}).then(state => {
      // response is the data from get(), not state.data
      const type = state.data.datatype || 'unknown';
      const name = state.curMapping.to_concept_code.replace(/[. -]/g, "_");
      const item = {
        map_type: state.curMapping.map_type,
        dhis2_uid: state.curMapping.from_concept_code,
        name: name,
        datatype: type, // Now type is the resolved value
      };
      console.log(item);
      // Append to mappingsList
      state.mappingsList.push(item);
      // Set in collections
      collections.set('global-almanach-dhis2', name, item);
      return state; // Return state for the next operation or each()
    }).catch(error => {
      console.error('Error fetching data for URL:', state.curMapping.from_concept_url, error);
      // Optionally handle the error, e.g., set a default type
      const name = state.curMapping.to_concept_code.replace(/[. -]/g, "_");
      const item = {
        map_type: state.curMapping.map_type,
        dhis2_uid: state.curMapping.from_concept_code,
        name: name,
        datatype: 'unknown', // Fallback on error
      };
      state.mappingsList.push(item);
      collections.set('global-almanach-dhis2', name, item);
      return state; // Continue processing other items
    })
);
1 Like