001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.util.Arrays;
016
017import org.eclipse.january.DatasetException;
018import org.eclipse.january.IMonitor;
019import org.eclipse.january.metadata.StatisticsMetadata;
020import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
021import org.slf4j.Logger;
022import org.slf4j.LoggerFactory;
023
024/**
025 * Generic container class for data that is compound in nature
026 * 
027 * Each subclass has an array of compound types, items of this array are composed of primitive types
028 * 
029 * Data items can be Complex, Vector, etc
030 * 
031 */
032public abstract class AbstractCompoundDataset extends AbstractDataset implements CompoundDataset {
033        // pin UID to base class
034        private static final long serialVersionUID = Dataset.serialVersionUID;
035
036        private static final Logger logger = LoggerFactory.getLogger(AbstractCompoundDataset.class);
037
038        protected int isize; // number of elements per item
039
040        @Override
041        public int getElementsPerItem() {
042                return isize;
043        }
044
045        @Override
046        protected int get1DIndex(final int i) {
047                int n = super.get1DIndex(i);
048                return stride == null ? isize * n : n;
049        }
050
051        @Override
052        protected int get1DIndex(final int i, final int j) {
053                int n = super.get1DIndex(i, j);
054                return stride == null ? isize * n : n;
055        }
056
057        @Override
058        protected int get1DIndexFromShape(final int[] n) {
059                return isize * super.get1DIndexFromShape(n);
060        }
061
062        @Override
063        public Dataset getUniqueItems() {
064                throw new UnsupportedOperationException("Cannot sort compound datasets");
065        }
066
067        @Override
068        public IndexIterator getIterator(final boolean withPosition) {
069                if (stride != null) {
070                        return base.getSize() == 1 ? 
071                                        (withPosition ? new PositionIterator(offset, shape) :  new SingleItemIterator(offset, size)) : new StrideIterator(isize, shape, stride, offset);
072                }
073                return withPosition ? getSliceIterator(null, null, null) :
074                        new ContiguousIterator(size, isize);
075        }
076
077        /**
078         * Get an iterator that picks out the chosen element from all items
079         * @param element to choose
080         * @return an iterator
081         */
082        public IndexIterator getIterator(int element) {
083                if (element < 0)
084                        element += isize;
085                if (element < 0 || element > isize) {
086                        logger.error("Invalid choice of element: {}/{}", element, isize);
087                        throw new IllegalArgumentException("Invalid choice of element: " + element + "/" + isize);
088                }
089
090                final IndexIterator it;
091                if (stride != null) {
092                        it = base.getSize() == 1 ? new SingleItemIterator(offset + element, size) : new StrideIterator(isize, shape, stride, offset, element);
093                } else {
094                        it = new ContiguousIterator(size, isize, element);
095                }
096
097                return it;
098        }
099
100        @Override
101        public IndexIterator getSliceIterator(SliceND slice) {
102                checkSliceND(slice);
103                return internalGetSliceIterator(slice);
104        }
105
106        /**
107         * @param slice to define iterator
108         * @return an slice iterator that operates like an IndexIterator
109         */
110        @Override
111        protected IndexIterator internalGetSliceIterator(SliceND slice) {
112                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
113                        return new NullIterator(shape, slice.getShape());
114                }
115                if (stride != null) {
116                        return new StrideIterator(isize, shape, stride, offset, slice);
117                }
118
119                return new SliceIterator(shape, size, isize, slice);
120        }
121
122        /**
123         * Constructor required for serialisation.
124         */
125        public AbstractCompoundDataset() {
126        }
127
128        @Override
129        public boolean equals(Object obj) {
130                if (!super.equals(obj)) {
131                        return false;
132                }
133
134                CompoundDataset other = (CompoundDataset) obj;
135                return isize == other.getElementsPerItem();
136        }
137
138        @Override
139        public int hashCode() {
140                return getCompoundStats().getHash(shape);
141        }
142
143        @Override
144        public CompoundDataset cast(boolean repeat, int dtype, int isize) {
145                return (CompoundDataset) super.cast(repeat, dtype, isize);
146        }
147
148        @Override
149        public CompoundDataset cast(int dtype) {
150                return (CompoundDataset) super.cast(dtype);
151        }
152
153        @Override
154        abstract public AbstractCompoundDataset clone();
155
156        @Override
157        public CompoundDataset flatten() {
158                return (CompoundDataset) super.flatten();
159        }
160
161        @Override
162        public CompoundDataset getBy1DIndex(IntegerDataset index) {
163                return (CompoundDataset) super.getBy1DIndex(index);
164        }
165
166        @Override
167        public CompoundDataset getByBoolean(Dataset selection) {
168                return (CompoundDataset) super.getByBoolean(selection);
169        }
170
171        @Override
172        public CompoundDataset getByIndexes(Object... indexes) {
173                return (CompoundDataset) super.getByIndexes(indexes);
174        }
175
176        @Override
177        public CompoundDataset getSlice(IMonitor mon, int[] start, int[] stop, int[] step) {
178                return (CompoundDataset) super.getSlice(mon, start, stop, step);
179        }
180
181        @Override
182        public CompoundDataset getSlice(IMonitor mon, Slice... slice) {
183                return (CompoundDataset) super.getSlice(mon, slice);
184        }
185
186        @Override
187        public CompoundDataset getSlice(IMonitor mon, SliceND slice) {
188                return (CompoundDataset) super.getSlice(mon, slice);
189        }
190
191        @Override
192        public CompoundDataset getSlice(int[] start, int[] stop, int[] step) {
193                return (CompoundDataset) super.getSlice(start, stop, step);
194        }
195
196        @Override
197        public CompoundDataset getSlice(Slice... slice) {
198                return (CompoundDataset) super.getSlice(slice);
199        }
200
201        @Override
202        public CompoundDataset getSlice(SliceND slice) {
203                return (CompoundDataset) super.getSlice(slice);
204        }
205
206        @Override
207        abstract public AbstractCompoundDataset getSlice(SliceIterator iterator);
208
209        @Override
210        public CompoundDataset getSliceView(int[] start, int[] stop, int[] step) {
211                return (CompoundDataset) super.getSliceView(start, stop, step);
212        }
213
214        @Override
215        public CompoundDataset getSliceView(Slice... slice) {
216                return (CompoundDataset) super.getSliceView(slice);
217        }
218
219        @Override
220        public CompoundDataset getSliceView(SliceND slice) {
221                return (CompoundDataset) super.getSliceView(slice);
222        }
223
224        @Override
225        public CompoundDataset getTransposedView(int... axes) {
226                return (CompoundDataset) super.getTransposedView(axes);
227        }
228
229        @Override
230        abstract public AbstractCompoundDataset getView(boolean deepCopyMetadata);
231
232        @Override
233        public CompoundDataset getBroadcastView(int... broadcastShape) {
234                return (CompoundDataset) super.getBroadcastView(broadcastShape);
235        }
236
237        @Override
238        public CompoundDataset ifloorDivide(Object o) {
239                return (CompoundDataset) super.ifloorDivide(o);
240        }
241
242        @Override
243        public CompoundDataset reshape(int... shape) {
244                return (CompoundDataset) super.reshape(shape);
245        }
246
247        @Override
248        public CompoundDataset setSlice(Object obj, int[] start, int[] stop, int[] step) {
249                return (CompoundDataset) super.setSlice(obj, start, stop, step);
250        }
251
252        @Override
253        public CompoundDataset setSlice(Object object, Slice... slice) {
254                return (CompoundDataset) super.setSlice(object, slice);
255        }
256
257        @Override
258        public CompoundDataset sort(Integer axis) {
259                throw new UnsupportedOperationException("Cannot sort dataset");
260        }
261
262        @Override
263        public CompoundDataset squeezeEnds() {
264                return (CompoundDataset) super.squeezeEnds();
265        }
266
267        @Override
268        public CompoundDataset squeeze() {
269                return (CompoundDataset) super.squeeze();
270        }
271
272        @Override
273        public CompoundDataset squeeze(boolean onlyFromEnd) {
274                return (CompoundDataset) super.squeeze(onlyFromEnd);
275        }
276
277        @Override
278        public CompoundDataset swapAxes(int axis1, int axis2) {
279                return (CompoundDataset) super.swapAxes(axis1, axis2);
280        }
281
282        @Override
283        public synchronized CompoundDataset synchronizedCopy() {
284                return clone();
285        }
286
287        @Override
288        public CompoundDataset transpose(int... axes) {
289                return (CompoundDataset) super.transpose(axes);
290        }
291
292        /**
293         * @since 2.0
294         * @return first value
295         */
296        abstract protected double getFirstValue();
297
298        abstract protected double getFirstValue(final int i);
299
300        abstract protected double getFirstValue(final int i, final int j);
301
302        abstract protected double getFirstValue(final int...pos);
303
304        @Override
305        public boolean getBoolean() {
306                return getFirstValue() != 0;
307        }
308
309        @Override
310        public boolean getBoolean(final int i) {
311                return getFirstValue(i) != 0;
312        }
313
314        @Override
315        public boolean getBoolean(final int i, final int j) {
316                return getFirstValue(i, j) != 0;
317        }
318
319        @Override
320        public boolean getBoolean(final int... pos) {
321                return getFirstValue(pos) != 0;
322        }
323
324        @Override
325        public byte getByte() {
326                return (byte) getFirstValue();
327        }
328
329        @Override
330        public byte getByte(final int i) {
331                return (byte) getFirstValue(i);
332        }
333
334        @Override
335        public byte getByte(final int i, final int j) {
336                return (byte) getFirstValue(i, j);
337        }
338
339        @Override
340        public byte getByte(final int... pos) {
341                return (byte) getFirstValue(pos);
342        }
343
344        @Override
345        public short getShort() {
346                return (short) getFirstValue();
347        }
348
349        @Override
350        public short getShort(final int i) {
351                return (short) getFirstValue(i);
352        }
353
354        @Override
355        public short getShort(final int i, final int j) {
356                return (short) getFirstValue(i, j);
357        }
358
359        @Override
360        public short getShort(final int... pos) {
361                return (short) getFirstValue(pos);
362        }
363
364        @Override
365        public int getInt() {
366                return (int) getFirstValue();
367        }
368
369        @Override
370        public int getInt(final int i) {
371                return (int) getFirstValue(i);
372        }
373
374        @Override
375        public int getInt(final int i, final int j) {
376                return (int) getFirstValue(i, j);
377        }
378
379        @Override
380        public int getInt(final int... pos) {
381                return (int) getFirstValue(pos);
382        }
383
384        @Override
385        public long getLong() {
386                return (long) getFirstValue();
387        }
388
389        @Override
390        public long getLong(final int i) {
391                return (long) getFirstValue(i);
392        }
393
394        @Override
395        public long getLong(final int i, final int j) {
396                return (long) getFirstValue(i, j);
397        }
398
399        @Override
400        public long getLong(final int... pos) {
401                return (long) getFirstValue(pos);
402        }
403
404        @Override
405        public float getFloat() {
406                return (float) getFirstValue();
407        }
408
409        @Override
410        public float getFloat(final int i) {
411                return (float) getFirstValue(i);
412        }
413
414        @Override
415        public float getFloat(final int i, final int j) {
416                return (float) getFirstValue(i, j);
417        }
418
419        @Override
420        public float getFloat(final int... pos) {
421                return (float) getFirstValue(pos);
422        }
423
424        @Override
425        public double getDouble() {
426                return getFirstValue();
427        }
428
429        @Override
430        public double getDouble(final int i) {
431                return getFirstValue(i);
432        }
433
434        @Override
435        public double getDouble(final int i, final int j) {
436                return getFirstValue(i, j);
437        }
438
439        @Override
440        public double getDouble(final int... pos) {
441                return getFirstValue(pos);
442        }
443
444        @Override
445        public void getDoubleArray(final double[] darray) {
446                getDoubleArrayAbs(getFirst1DIndex(), darray);
447        }
448
449        @Override
450        public void getDoubleArray(final double[] darray, final int i) {
451                getDoubleArrayAbs(get1DIndex(i), darray);
452        }
453
454        @Override
455        public void getDoubleArray(final double[] darray, final int i, final int j) {
456                getDoubleArrayAbs(get1DIndex(i, j), darray);
457        }
458
459        @Override
460        public void getDoubleArray(final double[] darray, final int... pos) {
461                getDoubleArrayAbs(get1DIndex(pos), darray);
462        }
463
464        /**
465         * @return statistics metadata
466         * @since 2.0
467         */
468        @SuppressWarnings("unchecked")
469        protected StatisticsMetadata<double[]> getCompoundStats() {
470                StatisticsMetadata<double[]> md = getFirstMetadata(StatisticsMetadata.class);
471                if (md == null || md.isDirty(this)) {
472                        md = new StatisticsMetadataImpl<double[]>();
473                        md.initialize(this);
474                        setMetadata(md);
475                }
476                return md;
477        }
478
479        @Override
480        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
481                logger.error("Cannot compare compound numbers");
482                throw new UnsupportedOperationException("Cannot compare compound numbers");
483        }
484
485        @Override
486        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
487                logger.error("Cannot compare compound numbers");
488                throw new UnsupportedOperationException("Cannot compare compound numbers");
489        }
490
491        @Override
492        public Number max(boolean... ignoreInvalids) {
493                logger.error("Cannot compare compound numbers");
494                throw new UnsupportedOperationException("Cannot compare compound numbers");
495        }
496
497        @Override
498        public CompoundDataset max(int axis, boolean... ignoreInvalids) {
499                logger.error("Cannot compare compound numbers");
500                throw new UnsupportedOperationException("Cannot compare compound numbers");
501        }
502
503        @Override
504        public Number min(boolean... ignoreInvalids) {
505                logger.error("Cannot compare compound numbers");
506                throw new UnsupportedOperationException("Cannot compare compound numbers");
507        }
508
509        @Override
510        public CompoundDataset min(int axis, boolean... ignoreInvalids) {
511                logger.error("Cannot compare compound numbers");
512                throw new UnsupportedOperationException("Cannot compare compound numbers");
513        }
514
515
516        @Override
517        public int[] maxPos(boolean... ignoreNaNs) {
518                logger.error("Cannot compare compound numbers");
519                throw new UnsupportedOperationException("Cannot compare compound numbers");
520        }
521
522        @Override
523        public int[] minPos(boolean... ignoreNaNs) {
524                logger.error("Cannot compare compound numbers");
525                throw new UnsupportedOperationException("Cannot compare compound numbers");
526        }
527
528        @Override
529        public CompoundDataset peakToPeak(int axis, boolean... ignoreInvalids) {
530                logger.error("Cannot compare compound numbers");
531                throw new UnsupportedOperationException("Cannot compare compound numbers");
532        }
533
534        @Override
535        public double[] maxItem() {
536                return getCompoundStats().getMaximum();
537        }
538
539        @Override
540        public double[] minItem() {
541                return getCompoundStats().getMinimum();
542        }
543
544        @Override
545        public Object mean(boolean... ignoreInvalids) {
546                return getCompoundStats().getMean();
547        }
548
549        @Override
550        public CompoundDataset mean(int axis, boolean... ignoreInvalids) {
551                return (CompoundDataset) super.mean(axis, ignoreInvalids);
552        }
553
554        @Override
555        public CompoundDataset product(int axis, boolean... ignoreInvalids) {
556                return (CompoundDataset) super.product(axis, ignoreInvalids);
557        }
558
559        @Override
560        public CompoundDataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
561                return (CompoundDataset) super.rootMeanSquare(axis, ignoreInvalids);
562        }
563
564        @Override
565        public CompoundDataset stdDeviation(int axis) {
566                return (CompoundDataset) super.stdDeviation(axis, false);
567        }
568
569        @Override
570        public CompoundDataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
571                return (CompoundDataset) super.stdDeviation(axis, isWholePopulation, ignoreInvalids);
572        }
573
574        @Override
575        public Object sum(boolean... ignoreInvalids) {
576                return getCompoundStats().getSum();
577        }
578
579        @Override
580        public CompoundDataset sum(int axis, boolean... ignoreInvalids) {
581                return (CompoundDataset) super.sum(axis, ignoreInvalids);
582        }
583
584        @Override
585        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
586                return getCompoundStats().getVariance(isWholePopulation, ignoreInvalids);
587        }
588
589        @Override
590        public CompoundDataset variance(int axis) {
591                return (CompoundDataset) super.variance(axis, false);
592        }
593
594        @Override
595        public CompoundDataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
596                return (CompoundDataset) super.variance(axis, isWholePopulation, ignoreInvalids);
597        }
598
599        @Override
600        public double rootMeanSquare(boolean... ignoreInvalids) {
601                StatisticsMetadata<double[]> stats = getCompoundStats();
602
603                double[] mean = stats.getMean(ignoreInvalids);
604                double result = 0;
605                for (int i = 0; i < isize; i++) {
606                        double m = mean[i];
607                        result += m * m;
608                }
609                return Math.sqrt(result + stats.getVariance(true));
610        }
611
612        /**
613         * @return error
614         */
615        private CompoundDataset getInternalError() {
616                ILazyDataset led = super.getErrors();
617                if (led == null)
618                        return null;
619
620                Dataset ed = null;
621                try {
622                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
623                } catch (DatasetException e) {
624                        logger.error("Could not get data from lazy dataset", e);
625                }
626
627                CompoundDataset ced; // ensure it has the same number of elements
628                if (!(ed instanceof CompoundDataset) || ed.getElementsPerItem() != isize) {
629                        ced = new CompoundDoubleDataset(isize, true, ed);
630                } else {
631                        ced = (CompoundDataset) ed;
632                }
633                
634                if (led != ced) {
635                        setErrors(ced); // set back
636                }
637                return ced;
638        }
639
640        @Override
641        public CompoundDataset getErrors() {
642                CompoundDataset ed = getInternalError();
643                if (ed == null)
644                        return null;
645
646                return ed.getBroadcastView(shape);
647        }
648
649        @Override
650        public double getError(final int i) {
651                return calcError(getInternalErrorArray(true, i));
652        }
653
654        @Override
655        public double getError(final int i, final int j) {
656                return calcError(getInternalErrorArray(true, i, j));
657        }
658
659        @Override
660        public double getError(final int... pos) {
661                return calcError(getInternalErrorArray(true, pos));
662        }
663
664        private double calcError(double[] es) {
665                if (es == null)
666                        return 0;
667
668                // assume elements are independent
669                double e = 0;
670                for (int k = 0; k < isize; k++) {
671                        e += es[k];
672                }
673
674                return Math.sqrt(e);
675        }
676
677        @Override
678        public double[] getErrorArray(final int i) {
679                return getInternalErrorArray(false, i);
680        }
681
682        @Override
683        public double[] getErrorArray(final int i, final int j) {
684                return getInternalErrorArray(false, i, j);
685        }
686
687        @Override
688        public double[] getErrorArray(final int... pos) {
689                return getInternalErrorArray(false, pos);
690        }
691
692        private Dataset getInternalError(final boolean squared) {
693                Dataset sed = squared ? getInternalSquaredError() : getInternalError();
694                if (sed == null)
695                        return null;
696
697                return sed.getBroadcastView(shape);
698        }
699
700        private double[] getInternalErrorArray(final boolean squared, final int i) {
701                Dataset sed = getInternalError(squared);
702                if (sed == null)
703                        return null;
704
705                double[] es;
706                if (sed instanceof CompoundDoubleDataset) {
707                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i);
708                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
709                                Arrays.fill(es, es[0]);
710                        }
711                } else {
712                        es = new double[isize];
713                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i));
714                }
715                return es;
716        }
717
718        private double[] getInternalErrorArray(final boolean squared, final int i, final int j) {
719                Dataset sed = getInternalError(squared);
720                if (sed == null)
721                        return null;
722
723                double[] es;
724                if (sed instanceof CompoundDoubleDataset) {
725                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i, j);
726                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
727                                Arrays.fill(es, es[0]);
728                        }
729                } else {
730                        es = new double[isize];
731                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i, j));
732                }
733                return es;
734        }
735
736        private double[] getInternalErrorArray(final boolean squared, final int... pos) {
737                Dataset sed = getInternalError(squared);
738                if (sed == null)
739                        return null;
740
741                double[] es = new double[isize];
742                if (sed instanceof CompoundDoubleDataset) {
743                        es = ((CompoundDoubleDataset) sed).getDoubleArray(pos);
744                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
745                                Arrays.fill(es, es[0]);
746                        }
747                } else {
748                        es = new double[isize];
749                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(pos));
750                }
751                return es;
752        }
753}
754