001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.annotation.Annotation;
017import java.lang.reflect.Array;
018import java.lang.reflect.Field;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.Collection;
022import java.util.concurrent.ConcurrentMap;
023import java.util.concurrent.ConcurrentHashMap;
024import java.util.List;
025import java.util.Map;
026
027import org.eclipse.january.DatasetException;
028import org.eclipse.january.MetadataException;
029import org.eclipse.january.metadata.Dirtiable;
030import org.eclipse.january.metadata.ErrorMetadata;
031import org.eclipse.january.metadata.IMetadata;
032import org.eclipse.january.metadata.MetadataFactory;
033import org.eclipse.january.metadata.MetadataType;
034import org.eclipse.january.metadata.Reshapeable;
035import org.eclipse.january.metadata.Sliceable;
036import org.eclipse.january.metadata.Transposable;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Common base for both lazy and normal dataset implementations
042 */
043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
044
045        private static final long serialVersionUID = 767926846438976050L;
046
047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
048
049        protected static boolean catchExceptions;
050        
051        static {
052                /**
053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
054                 */
055                catchExceptions = Boolean.getBoolean("run.in.eclipse");
056        }
057
058        protected String name = "";
059
060        /**
061         * The shape or dimensions of the dataset
062         */
063        protected int[] shape;
064
065        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
066
067        /**
068         * @return type of dataset item
069         */
070        abstract public int getDType();
071
072        @Override
073        public Class<?> getElementClass() {
074                return DTypeUtils.getElementClass(getDType());
075        }
076
077        @Override
078        public LazyDatasetBase clone() {
079                return null;
080        }
081
082        @Override
083        public boolean equals(Object obj) {
084                if (this == obj) {
085                        return true;
086                }
087                if (obj == null) {
088                        return false;
089                }
090                if (!getClass().equals(obj.getClass())) {
091                        return false;
092                }
093        
094                LazyDatasetBase other = (LazyDatasetBase) obj;
095                if (getDType() != other.getDType()) {
096                        return false;
097                }
098                if (getElementsPerItem() != other.getElementsPerItem()) {
099                        return false;
100                }
101                if (!Arrays.equals(shape, other.shape)) {
102                        return false;
103                }
104                return true;
105        }
106
107        @Override
108        public int hashCode() {
109                int hash = getDType() * 17 + getElementsPerItem();
110                int rank = shape.length;
111                for (int i = 0; i < rank; i++) {
112                        hash = hash*17 + shape[i];
113                }
114                return hash;
115        }
116
117        @Override
118        public String getName() {
119                return name;
120        }
121
122        @Override
123        public void setName(String name) {
124                this.name = name;
125        }
126
127        @Override
128        public int[] getShape() {
129                return shape.clone();
130        }
131
132        @Override
133        public int getRank() {
134                return shape.length;
135        }
136
137        /**
138         * Find first sub-interface of (or class that directly implements) MetadataType
139         * @param clazz
140         * @return sub-interface
141         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
142         */
143        @SuppressWarnings("unchecked")
144        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
145                if (clazz.equals(MetadataType.class)) {
146                        throw new IllegalArgumentException("Cannot accept MetadataType");
147                }
148
149                if (clazz.isInterface()) {
150                        return clazz;
151                }
152
153                if (clazz.isAnonymousClass()) { // special case
154                        Class<?> s = clazz.getSuperclass();
155                        if (!s.equals(Object.class)) {
156                                // only use super class if it is not an anonymous class of an interface
157                                clazz = (Class<? extends MetadataType>) s;
158                        }
159                }
160
161                for (Class<?> c : clazz.getInterfaces()) {
162                        if (c.equals(MetadataType.class)) {
163                                if (clazz.isAnonymousClass()) {
164                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
165                                }
166                                return clazz;
167                        }
168                        if (MetadataType.class.isAssignableFrom(c)) {
169                                return (Class<? extends MetadataType>) c;
170                        }
171                }
172
173                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
174                if (c != null) {
175                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
176                }
177
178                logger.error("Somehow the search for metadata type interface ended in a bad place");
179                assert false; // should not be able to get here!!!
180                return null;
181        }
182
183        @Override
184        public void setMetadata(MetadataType metadata) {
185                addMetadata(metadata, true);
186        }
187
188        @Override
189        public void addMetadata(MetadataType metadata) {
190                addMetadata(metadata, false);
191        }
192
193        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
194                if (metadata == null)
195                        return;
196
197                if (this.metadata == null) {
198                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
199                }
200
201                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
202                if (!this.metadata.containsKey(clazz)) {
203                        this.metadata.put(clazz, new ArrayList<MetadataType>());
204                } else if (clear) {
205                        this.metadata.get(clazz).clear();
206                }
207                this.metadata.get(clazz).add(metadata);
208
209                // add for special case of sub-interfaces of IMetadata
210                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
211                        clazz = IMetadata.class;
212                        if (!this.metadata.containsKey(clazz)) {
213                                this.metadata.put(clazz, new ArrayList<MetadataType>());
214                        } else if (clear) {
215                                this.metadata.get(clazz).clear();
216                        }
217                        this.metadata.get(clazz).add(metadata);
218                }
219        }
220
221        @Override
222        @Deprecated
223        public synchronized IMetadata getMetadata() {
224                return getFirstMetadata(IMetadata.class);
225        }
226
227        @SuppressWarnings("unchecked")
228        @Override
229        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
230                if (metadata == null)
231                        return null;
232
233                if (clazz == null) {
234                        List<S> all = new ArrayList<S>();
235                        for (Class<? extends MetadataType> c : metadata.keySet()) {
236                                all.addAll((Collection<S>) metadata.get(c));
237                        }
238                        return all;
239                }
240
241                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
242        }
243
244        @Override
245        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
246                try {
247                        List<S> ml = getMetadata(clazz);
248                        if (ml == null) return null;
249                        for (S t : ml) {
250                                if (clazz.isInstance(t)) return t;
251                        }
252                } catch (Exception e) {
253                        logger.error("Get metadata failed!",e);
254                }
255
256                return null;
257        }
258
259        @Override
260        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
261                if (metadata == null)
262                        return;
263
264                if (clazz == null) {
265                        metadata.clear();
266                        return;
267                }
268
269                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
270                if( list != null) {
271                        list.clear();
272                }
273        }
274
275        /**
276         * @since 2.0
277         */
278        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
279                return copyMetadata(metadata);
280        }
281
282        /**
283         * @since 2.0
284         */
285        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
286                if (metadata == null)
287                        return null;
288
289                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
290
291                for (Class<? extends MetadataType> c : metadata.keySet()) {
292                        List<MetadataType> l = metadata.get(c);
293                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
294                        map.put(c, nl);
295                        for (MetadataType m : l) {
296                                nl.add(m.clone());
297                        }
298                }
299                return map;
300        }
301
302        interface MetadatasetAnnotationOperation {
303                /**
304                 * Process value of given field
305                 * <p>
306                 * When the field is not a container then the returned value
307                 * may replace the old value
308                 * @param f given field
309                 * @param o value of field
310                 * @return transformed field
311                 */
312                Object processField(Field f, Object o);
313
314                /**
315                 * @return annotated class
316                 */
317                Class<? extends Annotation> getAnnClass();
318
319                /**
320                 * @param axis
321                 * @return number of dimensions to insert or remove
322                 */
323                int change(int axis);
324
325                /**
326                 * 
327                 * @return rank or -1 to match
328                 */
329                int getNewRank();
330
331                /**
332                 * Run on given lazy dataset
333                 * @param lz
334                 * @return 
335                 */
336                ILazyDataset run(ILazyDataset lz);
337        }
338
339        class MdsSlice implements MetadatasetAnnotationOperation {
340                private boolean asView;
341                private int[] start;
342                private int[] stop;
343                private int[] step;
344                private int[] oShape;
345                private long oSize;
346
347                public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) {
348                        this.asView = asView;
349                        this.start = start;
350                        this.stop = stop;
351                        this.step = step;
352                        this.oShape = oShape;
353                        oSize = ShapeUtils.calcLongSize(oShape);
354                }
355
356                @Override
357                public Object processField(Field field, Object o) {
358                        return o;
359                }
360
361                @Override
362                public Class<? extends Annotation> getAnnClass() {
363                        return Sliceable.class;
364                }
365
366                @Override
367                public int change(int axis) {
368                        return 0;
369                }
370
371                @Override
372                public int getNewRank() {
373                        return -1;
374                }
375
376                @Override
377                public ILazyDataset run(ILazyDataset lz) {
378                        int rank = lz.getRank();
379                        if (start.length != rank) {
380                                throw new IllegalArgumentException("Slice dimensions do not match dataset!");
381                        }
382
383                        int[] shape = lz.getShape();
384                        int[] stt;
385                        int[] stp;
386                        int[] ste;
387                        if (lz.getSize() == oSize) {
388                                stt = start;
389                                stp = stop;
390                                ste = step;
391                        } else {
392                                stt = start.clone();
393                                stp = stop.clone();
394                                ste = step.clone();
395                                for (int i = 0; i < rank; i++) {
396                                        if (shape[i] >= oShape[i]) continue;
397                                        if (shape[i] == 1) {
398                                                stt[i] = 0;
399                                                stp[i] = 1;
400                                                ste[1] = 1;
401                                        } else {
402                                                throw new IllegalArgumentException("Sliceable dataset has invalid size!");
403                                        }
404                                }
405                        }
406
407                        if (asView || (lz instanceof IDataset))
408                                return lz.getSliceView(stt, stp, ste);
409                        try {
410                                return lz.getSlice(stt, stp, ste);
411                        } catch (DatasetException e) {
412                                logger.error("Could not slice dataset in metadata", e);
413                                return null;
414                        }
415                }
416        }
417
418        class MdsReshape implements MetadatasetAnnotationOperation {
419                private boolean matchRank;
420                private int[] oldShape;
421                private int[] newShape;
422                boolean onesOnly;
423                int[] differences;
424
425                /*
426                 * if only ones then record differences (insertions and deletions)
427                 * 
428                 * if shape changing, find broadcasted dimensions and disallow
429                 * merging that include those dimensions
430                 */
431                public MdsReshape(final int[] oldShape, final int[] newShape) {
432                        this.oldShape = oldShape;
433                        this.newShape = newShape;
434                        differences = null;
435                }
436
437                @Override
438                public Object processField(Field field, Object o) {
439                        Annotation a = field.getAnnotation(Reshapeable.class);
440                        if (a != null) { // cannot be null
441                                matchRank = ((Reshapeable) a).matchRank();
442                        }
443                        return o;
444                }
445
446                @Override
447                public Class<? extends Annotation> getAnnClass() {
448                        return Reshapeable.class;
449                }       
450
451                @Override
452                public int change(int axis) {
453                        if (matchRank) {
454                                if (differences == null)
455                                        init();
456
457                                if (onesOnly) {
458                                        return differences[axis];
459                                }
460                                throw new UnsupportedOperationException("TODO support other shape operations");
461                        }
462                        return 0;
463                }
464
465                @Override
466                public int getNewRank() {
467                        return matchRank ? newShape.length : -1;
468                }
469
470                private void init() {
471                        int or = oldShape.length - 1;
472                        int nr = newShape.length - 1;
473                        if (or < 0 || nr < 0) { // zero-rank shapes
474                                onesOnly = true;
475                                differences = new int[1];
476                                differences[0] = or < 0 ? nr + 1 : or + 1;
477                                return;
478                        }
479                        int ob = 0;
480                        int nb = 0;
481                        onesOnly = true;
482                        do {
483                                while (oldShape[ob] == 1 && ob < or) {
484                                        ob++; // next non-unit dimension
485                                }
486                                while (newShape[nb] == 1 && nb < nr) {
487                                        nb++;
488                                }
489                                if (oldShape[ob++] != newShape[nb++]) {
490                                        onesOnly = false;
491                                        break;
492                                }
493                        } while (ob <= or && nb <= nr);
494
495                        ob = 0;
496                        nb = 0;
497                        differences = new int[or + 2];
498                        if (onesOnly) {
499                                // work out unit dimensions removed from or add to old
500                                int j = 0;
501                                do {
502                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
503                                                ob++;
504                                                nb++;
505                                        } else {
506                                                while (oldShape[ob] == 1 && ob < or) {
507                                                        ob++;
508                                                        differences[j]--;
509                                                }
510                                                while (newShape[nb] == 1 && nb < nr) {
511                                                        nb++;
512                                                        differences[j]++;
513                                                }
514                                        }
515                                        j++;
516                                } while (ob <= or && nb <= nr && j <= or);
517                                while (ob <= or && oldShape[ob] == 1) {
518                                        ob++;
519                                        differences[j]--;
520                                }
521                                while (nb <= nr && newShape[nb] == 1) {
522                                        nb++;
523                                        differences[j]++;
524                                }
525                        } else {
526                                if (matchRank) {
527                                        logger.error("Combining dimensions is currently not supported");
528                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
529                                }
530                                // work out mapping: contiguous dimensions can be grouped or split
531                                while (ob <= or && nb <= nr) {
532                                        int ol = oldShape[ob];
533                                        while (ol == 1 && ol <= or) {
534                                                ob++;
535                                                ol = oldShape[ob];
536                                        }
537                                        int oe = ob + 1;
538                                        int nl = newShape[nb];
539                                        while (nl == 1 && nl <= nr) {
540                                                nb++;
541                                                nl = newShape[nb];
542                                        }
543                                        int ne = nb + 1;
544                                        if (ol < nl) {
545                                                differences[ob] = 1;
546                                                do { // case where new shape combines several dimensions into one dimension
547                                                        if (oe == (or + 1)) {
548                                                                break;
549                                                        }
550                                                        differences[oe] = 1;
551                                                        ol *= oldShape[oe++];
552                                                } while (ol < nl);
553                                                differences[oe - 1] = oe - ob; // signal end with difference
554                                                if (nl != ol) {
555                                                        logger.error("Single dimension is incompatible with subshape");
556                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
557                                                }
558                                        } else if (ol > nl) {
559                                                do { // case where new shape spreads single dimension over several dimensions
560                                                        if (ne == (nr + 1)) {
561                                                                break;
562                                                        }
563                                                        nl *= newShape[ne++];
564                                                } while (nl < ol);
565                                                if (nl != ol) {
566                                                        logger.error("Subshape is incompatible with single dimension");
567                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
568                                                }
569
570                                        }
571
572                                        ob = oe;
573                                        nb = ne;
574                                }
575
576                        }
577                }
578
579                @Override
580                public ILazyDataset run(ILazyDataset lz) {
581                        if (differences == null)
582                                init();
583
584                        int[] lshape = lz.getShape();
585                        if (Arrays.equals(newShape, lshape)) {
586                                return lz;
587                        }
588                        int or = lz.getRank();
589                        int nr = newShape.length;
590                        int[] nshape = new int[nr];
591                        Arrays.fill(nshape, 1);
592                        if (onesOnly) {
593                                // ignore omit removed dimensions
594                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
595                                        int c = differences[i];
596                                        if (c == 0) {
597                                                nshape[di++] = lshape[si++];
598                                        } else if (c > 0) {
599                                                while (c-- > 0 && di < nr) {
600                                                        di++;
601                                                }
602                                        } else if (c < 0) {
603                                                si -= c; // remove dimensions by skipping forward in source array
604                                        }
605                                }
606                        } else {
607                                boolean[] broadcast = new boolean[or];
608                                for (int ob = 0; ob < or; ob++) {
609                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
610                                }
611                                int osize = lz.getSize();
612
613                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
614                                int ob = 0;
615                                int nsize = 1;
616                                for (int i = 0; i < nr; i++) {
617                                        if (ob < or && broadcast[ob]) {
618                                                if (differences[ob] != 0) {
619                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
620                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
621                                                }
622                                        } else {
623                                                nshape[i] = nsize < osize ? newShape[i] : 1;
624                                        }
625                                        nsize *= nshape[i];
626                                        ob++;
627                                }
628                        }
629
630                        ILazyDataset nlz = lz.getSliceView();
631                        if (lz instanceof Dataset) {
632                                nlz = ((Dataset) lz).reshape(nshape);
633                        } else {
634                                nlz = lz.getSliceView();
635                                nlz.setShape(nshape);
636                        }
637                        return nlz;
638                }
639        }
640
641        class MdsTranspose implements MetadatasetAnnotationOperation {
642                int[] map;
643
644                public MdsTranspose(final int[] axesMap) {
645                        map = axesMap;
646                }
647
648                @SuppressWarnings({ "rawtypes", "unchecked" })
649                @Override
650                public Object processField(Field f, Object o) {
651                        // reorder arrays and lists according the axes map
652                        if (o.getClass().isArray()) {
653                                int l = Array.getLength(o);
654                                if (l == map.length) {
655                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
656                                        for (int i = 0; i < l; i++) {
657                                                Array.set(narray, i, Array.get(o, map[i]));
658                                        }
659                                        for (int i = 0; i < l; i++) {
660                                                Array.set(o, i, Array.get(narray, i));
661                                        }
662                                }
663                        } else if (o instanceof List<?>) {
664                                List list = (List) o;
665                                int l = list.size();
666                                if (l == map.length) {
667                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
668                                        for (int i = 0; i < l; i++) {
669                                                Array.set(narray, i, list.get(map[i]));
670                                        }
671                                        list.clear();
672                                        for (int i = 0; i < l; i++) {
673                                                list.add(Array.get(narray, i));
674                                        }
675                                }
676                        }
677                        return o;
678                }
679
680                @Override
681                public Class<? extends Annotation> getAnnClass() {
682                        return Transposable.class;
683                }
684
685                @Override
686                public int change(int axis) {
687                        return 0;
688                }
689
690                @Override
691                public int getNewRank() {
692                        return -1;
693                }
694
695                @Override
696                public ILazyDataset run(ILazyDataset lz) {
697                        return lz.getTransposedView(map);
698                }
699        }
700
701        class MdsDirty implements MetadatasetAnnotationOperation {
702
703                @Override
704                public Object processField(Field f, Object o) {
705                        // throw exception if not boolean???
706                        Class<?> t = f.getType();
707                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
708                                if (o.equals(false)) {
709                                        o = true;
710                                }
711                        }
712                        return o;
713                }
714
715                @Override
716                public Class<? extends Annotation> getAnnClass() {
717                        return Dirtiable.class;
718                }
719
720                @Override
721                public int change(int axis) {
722                        return 0;
723                }
724
725                @Override
726                public int getNewRank() {
727                        return -1;
728                }
729
730                @Override
731                public ILazyDataset run(ILazyDataset lz) {
732                        return lz;
733                }
734        }
735
736        /**
737         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
738         * dataset after cloning the metadata
739         * @param asView if true then just a view
740         * @param slice
741         */
742        protected void sliceMetadata(boolean asView, final SliceND slice) {
743                processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true);
744        }
745
746        /**
747         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
748         * or setting the shape
749         * 
750         * @param newShape
751         */
752        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
753                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
754        }
755
756        /**
757         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
758         * dataset after cloning the metadata
759         * @param axesMap
760         */
761        protected void transposeMetadata(final int[] axesMap) {
762                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
763        }
764
765        /**
766         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
767         * @since 2.0
768         */
769        protected void dirtyMetadata() {
770                processAnnotatedMetadata(new MdsDirty(), true);
771        }
772
773        @SuppressWarnings("unchecked")
774        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
775                if (metadata == null)
776                        return;
777
778                for (Class<? extends MetadataType> c : metadata.keySet()) {
779                        for (MetadataType m : metadata.get(c)) {
780                                if (m == null)
781                                        continue;
782
783                                Class<? extends MetadataType> mc = m.getClass();
784                                do { // iterate over super-classes
785                                        processClass(op, m, mc, throwException);
786                                        Class<?> sclazz = mc.getSuperclass();
787                                        if (!MetadataType.class.isAssignableFrom(sclazz))
788                                                break;
789                                        mc = (Class<? extends MetadataType>) sclazz;
790                                } while (true);
791                        }
792                }
793        }
794
795        @SuppressWarnings({ "unchecked", "rawtypes" })
796        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
797                for (Field f : mc.getDeclaredFields()) {
798                        if (!f.isAnnotationPresent(op.getAnnClass()))
799                                continue;
800
801                        try {
802                                f.setAccessible(true);
803                                Object o = f.get(m);
804                                if (o == null)
805                                        continue;
806
807                                Object no = op.processField(f, o);
808                                if (no != o) {
809                                        f.set(m, no);
810                                        continue;
811                                }
812                                Object r = null;
813                                if (o instanceof ILazyDataset) {
814                                        try {
815                                                f.set(m, op.run((ILazyDataset) o));
816                                        } catch (Exception e) {
817                                                logger.error("Problem processing " + o, e);
818                                                if (!catchExceptions)
819                                                        throw e;
820                                        }
821                                } else if (o.getClass().isArray()) {
822                                        int l = Array.getLength(o);
823                                        if (l <= 0)
824                                                continue;
825
826                                        for (int i = 0; r == null && i < l; i++) {
827                                                r = Array.get(o, i);
828                                        }
829                                        int n = op.getNewRank();
830                                        if (r == null) {
831                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
832                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
833                                                }
834                                                continue;
835                                        }
836                                        if (n < 0)
837                                                n = l;
838                                        Object narray = Array.newInstance(r.getClass(), n);
839                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
840                                                int c = op.change(i);
841                                                if (c == 0) {
842                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
843                                                } else if (c > 0) {
844                                                        di += c; // add nulls by skipping forward in destination array
845                                                } else if (c < 0) {
846                                                        si -= c; // remove dimensions by skipping forward in source array
847                                                }
848                                        }
849                                        if (n == l) {
850                                                for (int i = 0; i < l; i++) {
851                                                        Array.set(o, i, Array.get(narray, i));
852                                                }
853                                        } else {
854                                                f.set(m, narray);
855                                        }
856                                } else if (o instanceof List<?>) {
857                                        List list = (List) o;
858                                        int l = list.size();
859                                        if (l <= 0)
860                                                continue;
861
862                                        for (int i = 0; r == null && i < l; i++) {
863                                                r = list.get(i);
864                                        }
865                                        int n = op.getNewRank();
866                                        if (r == null) {
867                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
868                                                        list.clear();
869                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
870                                                                list.add(null);
871                                                        }
872                                                }
873                                                continue;
874                                        }
875
876                                        if (n < 0)
877                                                n = l;
878                                        Object narray = Array.newInstance(r.getClass(), n);
879                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
880                                                int c = op.change(i);
881                                                if (c == 0) {
882                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
883                                                } else if (c > 0) {
884                                                        di += c; // add nulls by skipping forward in destination array
885                                                } else if (c < 0) {
886                                                        si -= c; // remove dimensions by skipping forward in source array
887                                                }
888                                        }
889                                        list.clear();
890                                        for (int i = 0; i < n; i++) {
891                                                list.add(Array.get(narray, i));
892                                        }
893                                } else if (o instanceof Map<?,?>) {
894                                        Map map = (Map) o;
895                                        for (Object k : map.keySet()) {
896                                                map.put(k, processObject(op, map.get(k)));
897                                        }
898                                }
899                        } catch (Exception e) {
900                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
901                                if (throwException)
902                                        throw new RuntimeException(e);
903                        }
904                }
905        }
906
907        @SuppressWarnings({ "unchecked", "rawtypes" })
908        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
909                if (o == null)
910                        return o;
911
912                if (o instanceof ILazyDataset) {
913                        try {
914                                return op.run((ILazyDataset) o);
915                        } catch (Exception e) {
916                                logger.error("Problem processing " + o, e);
917                                if (!catchExceptions)
918                                        throw e;
919                        }
920                } else if (o.getClass().isArray()) {
921                        int l = Array.getLength(o);
922                        for (int i = 0; i < l; i++) {
923                                Array.set(o, i, processObject(op, Array.get(o, i)));
924                        }
925                } else if (o instanceof List<?>) {
926                        List list = (List) o;
927                        for (int i = 0, imax = list.size(); i < imax; i++) {
928                                list.set(i, processObject(op, list.get(i)));
929                        }
930                } else if (o instanceof Map<?,?>) {
931                        Map map = (Map) o;
932                        for (Object k : map.keySet()) {
933                                map.put(k, processObject(op, map.get(k)));
934                        }
935                }
936                return o;
937        }
938
939        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
940                for (Class<? extends MetadataType> mc : oldMetadata.keySet()) {
941                        metadata.put(mc, oldMetadata.get(mc));
942                }
943        }
944
945        @SuppressWarnings("deprecation")
946        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
947                ILazyDataset d = null;
948                if (blob instanceof ILazyDataset) {
949                        d = (ILazyDataset) blob;
950                        if (d instanceof IDataset) {
951                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
952                                int is = ed.getElementsPerItem();
953                                if (is != 1 && is != getElementsPerItem()) {
954                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
955                                }
956                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
957                        } else if (!keepLazy) {
958                                final int is = getElementsPerItem();
959                                try {
960                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
961                                } catch (DatasetException e) {
962                                        logger.error("Could not get data from lazy dataset", e);
963                                        return null;
964                                }
965                        }
966                } else {
967                        final int is = getElementsPerItem();
968                        if (is == 1) {
969                                d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
970                        } else {
971                                try {
972                                        d = DatasetFactory.createFromObject(is, Dataset.ARRAYFLOAT64, blob);
973                                } catch (IllegalArgumentException e) { // if only single value supplied try again
974                                        d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
975                                }
976                        }
977                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
978                                d.setShape(shape.clone());
979                        }
980                }
981                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
982                d.setShape(s.get(0));
983
984                return d;
985        }
986
987        @Override
988        public void setErrors(Serializable errors) {
989                if (shape == null) {
990                        throw new IllegalArgumentException("Cannot set errors for null dataset");
991                }
992                if (errors == null) {
993                        clearMetadata(ErrorMetadata.class);
994                        return;
995                }
996                if (errors == this) {
997                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
998                        return;
999                }
1000
1001                ILazyDataset errorData = createFromSerializable(errors, true);
1002
1003                ErrorMetadata emd = getErrorMetadata();
1004                if (emd == null) {
1005                        try {
1006                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1007                                setMetadata(emd);
1008                        } catch (MetadataException me) {
1009                                logger.error("Could not create metadata", me);
1010                        }
1011                }
1012                emd.setError(errorData);
1013        }
1014
1015        protected ErrorMetadata getErrorMetadata() {
1016                try {
1017                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
1018                        if (el != null && !el.isEmpty()) {
1019                                 return el.get(0);
1020                        }
1021                } catch (Exception e) {
1022                }
1023                return null;
1024        }
1025
1026        @Override
1027        public ILazyDataset getErrors() {
1028                ErrorMetadata emd = getErrorMetadata();
1029                return emd == null ? null : emd.getError();
1030        }
1031
1032        @Override
1033        public boolean hasErrors() {
1034                return LazyDatasetBase.this.getErrors() != null;
1035        }
1036
1037        /**
1038         * Check permutation axes
1039         * @param shape
1040         * @param axes
1041         * @return cleaned up axes or null if trivial
1042         */
1043        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
1044                int rank = shape.length;
1045        
1046                if (axes == null || axes.length == 0) {
1047                        axes = new int[rank];
1048                        for (int i = 0; i < rank; i++) {
1049                                axes[i] = rank - 1 - i;
1050                        }
1051                }
1052
1053                if (axes.length != rank) {
1054                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
1055                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
1056                }
1057        
1058                // check all permutation values are within bounds
1059                for (int d : axes) {
1060                        if (d < 0 || d >= rank) {
1061                                logger.error("axis permutation contains element {} outside rank of dataset", d);
1062                                throw new IllegalArgumentException("axis permutation contains element outside rank of dataset");
1063                        }
1064                }
1065        
1066                // check for a valid permutation (is this an unnecessary restriction?)
1067                int[] perm = axes.clone();
1068                Arrays.sort(perm);
1069
1070                for (int i = 0; i < rank; i++) {
1071                        if (perm[i] != i) {
1072                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
1073                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");   
1074                        }
1075                }
1076
1077                if (Arrays.equals(axes, perm))
1078                        return null; // signal identity or trivial permutation
1079
1080                return axes;
1081        }
1082}